xonecas.io
53129
Pa

PAT / commit ff2581a

feat: stream ollama response

xonecas2025-12-21 ff2581a45fbca7fe4a61aa0add0f08727abeaba0 parent ff3718b

7 files changed

drizzle/0007_spotty_maestro.sql+6−0
@@ -0,0 +1,6 @@
1+ALTER TABLE "threads" ALTER COLUMN "model" DROP DEFAULT;--> statement-breakpoint
2+ALTER TABLE "messages" ADD COLUMN "thinking" text;--> statement-breakpoint
3+ALTER TABLE "messages" DROP COLUMN "model";--> statement-breakpoint
4+ALTER TABLE "messages" DROP COLUMN "max_prompt_length";--> statement-breakpoint
5+ALTER TABLE "messages" DROP COLUMN "generation_time_ms";--> statement-breakpoint
6+ALTER TABLE "messages" DROP COLUMN "tool_call_counts";
drizzle/meta/0007_snapshot.json+140−0
@@ -0,0 +1,140 @@
1+{
2+ "id": "ca5189e9-3478-4d36-8016-f20ddea2c568",
3+ "prevId": "58baed47-8a10-4003-81cf-3cc9854ac3cb",
4+ "version": "7",
5+ "dialect": "postgresql",
6+ "tables": {
7+ "public.messages": {
8+ "name": "messages",
9+ "schema": "",
10+ "columns": {
11+ "id": {
12+ "name": "id",
13+ "type": "serial",
14+ "primaryKey": true,
15+ "notNull": true
16+ },
17+ "thread_id": {
18+ "name": "thread_id",
19+ "type": "integer",
20+ "primaryKey": false,
21+ "notNull": true
22+ },
23+ "role": {
24+ "name": "role",
25+ "type": "text",
26+ "primaryKey": false,
27+ "notNull": true
28+ },
29+ "content": {
30+ "name": "content",
31+ "type": "text",
32+ "primaryKey": false,
33+ "notNull": true
34+ },
35+ "thinking": {
36+ "name": "thinking",
37+ "type": "text",
38+ "primaryKey": false,
39+ "notNull": false
40+ },
41+ "created_at": {
42+ "name": "created_at",
43+ "type": "timestamp with time zone",
44+ "primaryKey": false,
45+ "notNull": true,
46+ "default": "now()"
47+ }
48+ },
49+ "indexes": {},
50+ "foreignKeys": {
51+ "messages_thread_id_threads_id_fk": {
52+ "name": "messages_thread_id_threads_id_fk",
53+ "tableFrom": "messages",
54+ "tableTo": "threads",
55+ "columnsFrom": [
56+ "thread_id"
57+ ],
58+ "columnsTo": [
59+ "id"
60+ ],
61+ "onDelete": "cascade",
62+ "onUpdate": "no action"
63+ }
64+ },
65+ "compositePrimaryKeys": {},
66+ "uniqueConstraints": {},
67+ "policies": {},
68+ "checkConstraints": {},
69+ "isRLSEnabled": false
70+ },
71+ "public.threads": {
72+ "name": "threads",
73+ "schema": "",
74+ "columns": {
75+ "id": {
76+ "name": "id",
77+ "type": "serial",
78+ "primaryKey": true,
79+ "notNull": true
80+ },
81+ "title": {
82+ "name": "title",
83+ "type": "text",
84+ "primaryKey": false,
85+ "notNull": false
86+ },
87+ "model": {
88+ "name": "model",
89+ "type": "text",
90+ "primaryKey": false,
91+ "notNull": true
92+ },
93+ "max_prompt_length": {
94+ "name": "max_prompt_length",
95+ "type": "integer",
96+ "primaryKey": false,
97+ "notNull": false
98+ },
99+ "user_prompt": {
100+ "name": "user_prompt",
101+ "type": "text",
102+ "primaryKey": false,
103+ "notNull": false
104+ },
105+ "created_at": {
106+ "name": "created_at",
107+ "type": "timestamp with time zone",
108+ "primaryKey": false,
109+ "notNull": true,
110+ "default": "now()"
111+ },
112+ "updated_at": {
113+ "name": "updated_at",
114+ "type": "timestamp with time zone",
115+ "primaryKey": false,
116+ "notNull": true,
117+ "default": "now()"
118+ }
119+ },
120+ "indexes": {},
121+ "foreignKeys": {},
122+ "compositePrimaryKeys": {},
123+ "uniqueConstraints": {},
124+ "policies": {},
125+ "checkConstraints": {},
126+ "isRLSEnabled": false
127+ }
128+ },
129+ "enums": {},
130+ "schemas": {},
131+ "sequences": {},
132+ "roles": {},
133+ "policies": {},
134+ "views": {},
135+ "_meta": {
136+ "columns": {},
137+ "schemas": {},
138+ "tables": {}
139+ }
140+}
drizzle/meta/_journal.json+7−0
@@ -50,6 +50,13 @@
5050 "when": 1766148194532,
5151 "tag": "0006_productive_smasher",
5252 "breakpoints": true
53+ },
54+ {
55+ "idx": 7,
56+ "version": "7",
57+ "when": 1766350897443,
58+ "tag": "0007_spotty_maestro",
59+ "breakpoints": true
5360 }
5461 ]
5562 }
src/lib/chat.ts+69−61
@@ -1,16 +1,63 @@
1-import { debug } from "@/src/lib/debug";
1+import { db } from "@/src/lib/db";
2+import { messages, threads } from "@/src/lib/db/schema";
23 import { createMessage } from "@/src/lib/messages";
3-import {
4- duckDuckGoTool,
5- executeToolCall,
6- fetchPageTool,
7- weatherTool,
8- webSearchTool,
9-} from "@/src/lib/tools";
4+import { executeToolCall, fetchPageTool, weatherTool, webSearchTool } from "@/src/lib/tools";
105 import { asc, eq } from "drizzle-orm";
11-import ollama, { type Tool } from "ollama";
12-import { db } from "./db";
13-import { messages, threads } from "./db/schema";
6+import ollama, { type Tool, type ToolCall } from "ollama";
7+
8+type MessageHistory = {
9+ role: "user" | "assistant" | "system" | "tool";
10+ content: string;
11+ thinking?: string;
12+}[];
13+
14+async function agentLoop(model: string, messageHistory: MessageHistory) {
15+ const tools: Tool[] = [weatherTool, fetchPageTool, webSearchTool];
16+ const newMessages: MessageHistory = [];
17+ let maxIterations = 6;
18+
19+ while (true) {
20+ maxIterations--;
21+
22+ const stream = await ollama.chat({
23+ model: model,
24+ messages: [...messageHistory, ...newMessages],
25+ tools: maxIterations > 0 ? tools : undefined,
26+ think: true,
27+ stream: true,
28+ });
29+
30+ let thinking = "";
31+ let content = "";
32+ const calls: ToolCall[] = [];
33+
34+ for await (const chunk of stream) {
35+ if (chunk.message.thinking) {
36+ thinking += chunk.message;
37+ }
38+ if (chunk.message.content) {
39+ content += chunk.message.content;
40+ }
41+ if (chunk.message.tool_calls?.length) {
42+ calls.push(...chunk.message.tool_calls);
43+ }
44+ }
45+
46+ if (content) {
47+ newMessages.push({ role: "assistant", content: content, thinking: thinking });
48+ }
49+ if (calls.length) {
50+ for (const call of calls) {
51+ const toolResponse = await executeToolCall(call);
52+ newMessages.push({ role: "tool", content: `[${call.function.name}] ${toolResponse}` });
53+ }
54+ } else {
55+ break;
56+ }
57+ }
58+
59+ return newMessages;
60+}
1461
1562 export async function generateResponse(threadId: number) {
1663 if (!threadId) {
@@ -24,63 +71,24 @@ export async function generateResponse(threadId: number) {
2471 }
2572
2673 // Get all of the messages for this thread:
27- const messagesList = await db
28- .select()
74+ const messagesList = (await db
75+ .select({
76+ role: messages.role,
77+ content: messages.content,
78+ thinking: messages.thinking,
79+ })
2980 .from(messages)
3081 .where(eq(messages.threadId, threadId))
31- .orderBy(asc(messages.createdAt));
82+ .orderBy(asc(messages.createdAt))) as MessageHistory;
3283
3384 if (messagesList.length === 0) {
3485 throw new Error("No messages found for this thread");
3586 }
3687
37- const tools: Tool[] = [duckDuckGoTool, weatherTool, fetchPageTool, webSearchTool];
38- const chatMessages = messagesList.map((message) => ({
39- role: message.role,
40- content: message.content,
41- }));
42- let iterations = 0;
43- let toolCallsExist = true;
44-
45- // Repeatedly generate model responses until no tool calls remain
46- while (toolCallsExist) {
47- iterations++;
88+ const messageHistory = await agentLoop(thread[0].model, messagesList);
4889
49- const response = await ollama.chat({
50- model: thread[0].model,
51- messages: chatMessages,
52- tools: iterations >= 10 ? undefined : tools, // Avoid looping forever
53- think: true,
54- });
55-
56- if (response.message.thinking) {
57- debug("Thinking:", response.message.thinking);
58- }
59-
60- if (response.message?.tool_calls && response.message.tool_calls.length > 0) {
61- for (const toolCall of response.message.tool_calls) {
62- try {
63- const toolResponse = await executeToolCall(toolCall);
64- await createMessage(`${toolCall.function.name}: ${toolResponse}`, "tool", threadId);
65- chatMessages.push({
66- role: "tool",
67- content: toolResponse,
68- });
69- } catch (error) {
70- const content = `${toolCall.function.name}: Error: ${error instanceof Error ? error.message : "Unknown error"}`;
71- await createMessage(content, "tool", threadId);
72- chatMessages.push({
73- role: "tool",
74- content: content,
75- });
76-
77- debug(content);
78- }
79- }
80- } else {
81- toolCallsExist = false;
82- // insert the final response into the thread:
83- await createMessage(response.message.content, "assistant", threadId);
84- }
90+ // Update the thread with the new message history:
91+ for (const message of messageHistory) {
92+ await createMessage(message.content, message.role, threadId);
8593 }
8694 }
src/lib/db/schema.ts+2−5
@@ -5,7 +5,7 @@ import { integer, pgTable, serial, text, timestamp } from "drizzle-orm/pg-core";
55 export const threads = pgTable("threads", {
66 id: serial("id").primaryKey(),
77 title: text("title"),
8- model: text("model").notNull().default("gpt-oss"),
8+ model: text("model").notNull(),
99 maxPromptLength: integer("max_prompt_length"), // null, 1024, or 4096
1010 userPrompt: text("user_prompt"), // User-defined prompt to include in system prompt
1111 createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
@@ -19,11 +19,8 @@ export const messages = pgTable("messages", {
1919 .references(() => threads.id, { onDelete: "cascade" }),
2020 role: text("role").notNull(), // "user" or "assistant"
2121 content: text("content").notNull(),
22- model: text("model"), // Model used to generate this message (for assistant messages)
23- maxPromptLength: integer("max_prompt_length"), // Prompt size used to generate this message (for assistant messages) - null, 1024, or 4096
22+ thinking: text("thinking"), // Thinking (for assistant messages)
2423 createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
25- generationTimeMs: integer("generation_time_ms"), // Time taken to generate response (for assistant messages)
26- toolCallCounts: text("tool_call_counts"), // JSON string of tool name counts, e.g. {"query_weather": 2, "query_web_search": 1}
2724 });
2825
2926 // Define relations
src/lib/tools/duckduckgo.ts+0−85
@@ -1,85 +0,0 @@
1-import { getCache, setCache } from "@/src/lib/cache";
2-import { createRateLimiter } from "@/src/lib/ratelimit";
3-
4-const CACHE_TTL_MS = 6 * 60 * 60 * 1000; // 6 hours
5-const duckDuckGoRateLimiter = createRateLimiter({
6- maxRequests: 500,
7- windowMs: 24 * 60 * 60 * 1000, // 24 hours in milliseconds
8- identifier: "duckduckgo",
9-});
10-
11-/**
12- * Queries DuckDuckGo Instant Answer API. Results cached 6h.
13- */
14-export async function queryDuckDuckGo(query: string): Promise<string> {
15- // Check cache first
16- const cacheKey = `duckduckgo:${query.toLowerCase().trim()}`;
17- const cached = await getCache<string>(cacheKey);
18- if (cached !== null) {
19- return cached;
20- }
21-
22- const rateLimitCheck = await duckDuckGoRateLimiter.check();
23- if (!rateLimitCheck.allowed) {
24- throw new Error("Service temporarily unavailable");
25- }
26-
27- await duckDuckGoRateLimiter.increment();
28- const url = "https://api.duckduckgo.com/";
29- const params = new URLSearchParams({
30- q: query,
31- format: "json",
32- no_html: "1",
33- skip_disambig: "1",
34- });
35-
36- const response = await fetch(`${url}?${params.toString()}`);
37- if (!response.ok) {
38- throw new Error("Service error");
39- }
40-
41- const data = await response.json();
42-
43- const parts: string[] = [];
44- if (data.Answer) parts.push(data.Answer);
45- if (data.AbstractText) parts.push(data.AbstractText);
46- if (data.Definition) parts.push(data.Definition);
47- if (data.Heading && !parts.length) parts.push(data.Heading);
48-
49- if (parts.length === 0) {
50- const result = `No answer found for: ${query}`;
51- await setCache(cacheKey, result, CACHE_TTL_MS);
52- return result;
53- }
54-
55- let result = parts[0];
56- if (data.AbstractURL) result += ` (${data.AbstractURL})`;
57- if (data.RelatedTopics?.length > 0) {
58- const topics = data.RelatedTopics.slice(0, 2)
59- .map((t: { Text?: string }) => t.Text)
60- .filter(Boolean)
61- .join(", ");
62- if (topics) result += ` | Related: ${topics}`;
63- }
64-
65- await setCache(cacheKey, result, CACHE_TTL_MS);
66- return result;
67-}
68-
69-export const duckDuckGoTool = {
70- type: "function" as const,
71- function: {
72- name: "query_duckduckgo",
73- description: "Get quick info from DuckDuckGo about topics, people, places, or concepts.",
74- parameters: {
75- type: "object",
76- properties: {
77- query: {
78- type: "string",
79- description: "Search query",
80- },
81- },
82- required: ["query"],
83- },
84- },
85-};
src/lib/tools/index.ts+0−4
@@ -1,12 +1,10 @@
11 /* personal-assistant-thing/src/lib/ollama/tools/index.ts */
22
3-import { queryDuckDuckGo } from "@/src/lib/tools/duckduckgo";
43 import { fetchPage } from "@/src/lib/tools/fetchpage";
54 import { queryWeather } from "@/src/lib/tools/weather";
65 import { queryWebSearch } from "@/src/lib/tools/websearch";
76 import type { ToolCall } from "ollama";
87
9-export { duckDuckGoTool, queryDuckDuckGo } from "@/src/lib/tools/duckduckgo";
108 export { fetchPage, fetchPageTool } from "@/src/lib/tools/fetchpage";
119 export { queryWeather, weatherTool } from "@/src/lib/tools/weather";
1210 export { queryWebSearch, webSearchTool } from "@/src/lib/tools/websearch";
@@ -23,8 +21,6 @@ export async function executeToolCall(toolCall: ToolCall) {
2321 return await fetchPage(toolCall.function.arguments.url);
2422 case "query_web_search":
2523 return await queryWebSearch(toolCall.function.arguments.query);
26- case "query_duckduckgo":
27- return await queryDuckDuckGo(toolCall.function.arguments.query);
2824 default:
2925 throw new Error(`Unknown tool: ${toolCall.function.name}`);
3026 }