xonecas.io
53129
Pa

PAT / commit b49cd51

Merge pull request #2 from sacenox/feat-streaming-response

feat: stream ollama response

xonecas2025-12-22 b49cd5160a91b26f942f3199ddb7e84fcec0d938 parent ff3718bparent d5ae5c9

16 files changed

app/api/threads/[id]/messages/route.ts+13−4
@@ -1,4 +1,4 @@
1-import { generateResponse } from "@/src/lib/chat";
1+import { generateResponse, streamResponse } from "@/src/lib/chat";
22 import { db } from "@/src/lib/db";
33 import { messages } from "@/src/lib/db/schema";
44 import { debug } from "@/src/lib/debug";
@@ -63,10 +63,19 @@ export async function POST(request: Request, { params }: { params: Promise<{ id:
6363
6464 await createMessage(message, "user", threadId);
6565
66+ const stream = streamResponse(async (enqueue) => {
67+ if (threadId) {
68+ await generateResponse(threadId, enqueue);
69+ }
70+ });
6671 // Generate a response from the model.
67- await generateResponse(threadId);
68-
69- return NextResponse.json({ message: message });
72+ return new Response(stream, {
73+ headers: {
74+ "Content-Type": "text/event-stream",
75+ "Cache-Control": "no-cache",
76+ Connection: "keep-alive",
77+ },
78+ });
7079 } catch (err) {
7180 debug(
7281 `[Messages API] Error creating message for thread ${threadId ?? "unknown"}:`,
drizzle/0007_spotty_maestro.sql+6−0
@@ -0,0 +1,6 @@
1+ALTER TABLE "threads" ALTER COLUMN "model" DROP DEFAULT;--> statement-breakpoint
2+ALTER TABLE "messages" ADD COLUMN "thinking" text;--> statement-breakpoint
3+ALTER TABLE "messages" DROP COLUMN "model";--> statement-breakpoint
4+ALTER TABLE "messages" DROP COLUMN "max_prompt_length";--> statement-breakpoint
5+ALTER TABLE "messages" DROP COLUMN "generation_time_ms";--> statement-breakpoint
6+ALTER TABLE "messages" DROP COLUMN "tool_call_counts";
drizzle/meta/0007_snapshot.json+140−0
@@ -0,0 +1,140 @@
1+{
2+ "id": "ca5189e9-3478-4d36-8016-f20ddea2c568",
3+ "prevId": "58baed47-8a10-4003-81cf-3cc9854ac3cb",
4+ "version": "7",
5+ "dialect": "postgresql",
6+ "tables": {
7+ "public.messages": {
8+ "name": "messages",
9+ "schema": "",
10+ "columns": {
11+ "id": {
12+ "name": "id",
13+ "type": "serial",
14+ "primaryKey": true,
15+ "notNull": true
16+ },
17+ "thread_id": {
18+ "name": "thread_id",
19+ "type": "integer",
20+ "primaryKey": false,
21+ "notNull": true
22+ },
23+ "role": {
24+ "name": "role",
25+ "type": "text",
26+ "primaryKey": false,
27+ "notNull": true
28+ },
29+ "content": {
30+ "name": "content",
31+ "type": "text",
32+ "primaryKey": false,
33+ "notNull": true
34+ },
35+ "thinking": {
36+ "name": "thinking",
37+ "type": "text",
38+ "primaryKey": false,
39+ "notNull": false
40+ },
41+ "created_at": {
42+ "name": "created_at",
43+ "type": "timestamp with time zone",
44+ "primaryKey": false,
45+ "notNull": true,
46+ "default": "now()"
47+ }
48+ },
49+ "indexes": {},
50+ "foreignKeys": {
51+ "messages_thread_id_threads_id_fk": {
52+ "name": "messages_thread_id_threads_id_fk",
53+ "tableFrom": "messages",
54+ "tableTo": "threads",
55+ "columnsFrom": [
56+ "thread_id"
57+ ],
58+ "columnsTo": [
59+ "id"
60+ ],
61+ "onDelete": "cascade",
62+ "onUpdate": "no action"
63+ }
64+ },
65+ "compositePrimaryKeys": {},
66+ "uniqueConstraints": {},
67+ "policies": {},
68+ "checkConstraints": {},
69+ "isRLSEnabled": false
70+ },
71+ "public.threads": {
72+ "name": "threads",
73+ "schema": "",
74+ "columns": {
75+ "id": {
76+ "name": "id",
77+ "type": "serial",
78+ "primaryKey": true,
79+ "notNull": true
80+ },
81+ "title": {
82+ "name": "title",
83+ "type": "text",
84+ "primaryKey": false,
85+ "notNull": false
86+ },
87+ "model": {
88+ "name": "model",
89+ "type": "text",
90+ "primaryKey": false,
91+ "notNull": true
92+ },
93+ "max_prompt_length": {
94+ "name": "max_prompt_length",
95+ "type": "integer",
96+ "primaryKey": false,
97+ "notNull": false
98+ },
99+ "user_prompt": {
100+ "name": "user_prompt",
101+ "type": "text",
102+ "primaryKey": false,
103+ "notNull": false
104+ },
105+ "created_at": {
106+ "name": "created_at",
107+ "type": "timestamp with time zone",
108+ "primaryKey": false,
109+ "notNull": true,
110+ "default": "now()"
111+ },
112+ "updated_at": {
113+ "name": "updated_at",
114+ "type": "timestamp with time zone",
115+ "primaryKey": false,
116+ "notNull": true,
117+ "default": "now()"
118+ }
119+ },
120+ "indexes": {},
121+ "foreignKeys": {},
122+ "compositePrimaryKeys": {},
123+ "uniqueConstraints": {},
124+ "policies": {},
125+ "checkConstraints": {},
126+ "isRLSEnabled": false
127+ }
128+ },
129+ "enums": {},
130+ "schemas": {},
131+ "sequences": {},
132+ "roles": {},
133+ "policies": {},
134+ "views": {},
135+ "_meta": {
136+ "columns": {},
137+ "schemas": {},
138+ "tables": {}
139+ }
140+}
drizzle/meta/_journal.json+7−0
@@ -50,6 +50,13 @@
5050 "when": 1766148194532,
5151 "tag": "0006_productive_smasher",
5252 "breakpoints": true
53+ },
54+ {
55+ "idx": 7,
56+ "version": "7",
57+ "when": 1766350897443,
58+ "tag": "0007_spotty_maestro",
59+ "breakpoints": true
5360 }
5461 ]
5562 }
src/components/App.tsx+9−14
@@ -1,30 +1,28 @@
11 "use client";
22
3-import MessageEditor from "@/src/components/MessageEditor";
4-import MessageList from "@/src/components/MessageList";
3+import Main from "@/src/components/Main";
54 import Sidebar from "@/src/components/Sidebar";
6-import Welcome from "@/src/components/Welcome";
75 import { useModels, type Model } from "@/src/hooks/api/useModels";
86 import { useLocalStorage } from "@/src/hooks/useLocalStorage";
97 import { useTheme } from "@/src/hooks/useTheme";
10-import { createContext, useContext, useState } from "react";
8+import { createContext, Dispatch, SetStateAction, useContext, useState } from "react";
119
1210 export type AppContextType = {
1311 selectedModel: Model | null;
14- setSelectedModel: (model: Model) => void;
12+ setSelectedModel: Dispatch<SetStateAction<Model | null>>;
1513 maxPromptLength: number | null;
16- setMaxPromptLength: (length: number | null) => void;
14+ setMaxPromptLength: Dispatch<SetStateAction<number | null>>;
1715 userPrompt: string;
18- setUserPrompt: (prompt: string) => void;
16+ setUserPrompt: Dispatch<SetStateAction<string>>;
1917 models: Model[];
2018 isModelsLoading: boolean;
2119 modelsError: Error | null;
2220 selectedThreadId: number | null;
23- setSelectedThreadId: (threadId: number | null) => void;
21+ setSelectedThreadId: Dispatch<SetStateAction<number | null>>;
2422 showSystemMessages: boolean;
25- setShowSystemMessages: (show: boolean) => void;
23+ setShowSystemMessages: Dispatch<SetStateAction<boolean>>;
2624 showToolMessages: boolean;
27- setShowToolMessages: (show: boolean) => void;
25+ setShowToolMessages: Dispatch<SetStateAction<boolean>>;
2826 themeMode: "device" | "dark" | "light";
2927 handleThemeChange: (mode: "device" | "dark" | "light") => void;
3028 };
@@ -85,10 +83,7 @@ export default function App() {
8583 }}
8684 >
8785 <div className="flex h-screen overflow-hidden bg-neutral-100 text-neutral-800 dark:bg-neutral-900 dark:text-neutral-200">
88- <div className="relative flex-1 overflow-y-auto pb-48">
89- {selectedThreadId ? <MessageList /> : <Welcome />}
90- <MessageEditor />
91- </div>
86+ <Main />
9287 <Sidebar />
9388 </div>
9489 </AppContext.Provider>
src/components/Main.tsx+43−0
@@ -0,0 +1,43 @@
1+"use client";
2+
3+import { useAppContext } from "@/src/components/App";
4+import MessageEditor from "@/src/components/MessageEditor";
5+import MessageList from "@/src/components/MessageList";
6+import Welcome from "@/src/components/Welcome";
7+import { useThreadMessages } from "@/src/hooks/api/useThreadMessages";
8+import { Message } from "@/src/lib/db/schema";
9+import { useCallback, useState } from "react";
10+
11+export default function Main() {
12+ const [loadedMessages, setLoadedMessages] = useState<Message[]>([]);
13+ const { selectedThreadId, showSystemMessages, showToolMessages } = useAppContext();
14+ const optionalRoles = [
15+ ...(showSystemMessages ? ["system"] : []),
16+ ...(showToolMessages ? ["tool"] : []),
17+ ];
18+
19+ const handleSetLoadedMessages = useCallback((messages: Message[]) => {
20+ setLoadedMessages(messages);
21+ }, []);
22+
23+ const { isLoading: isLoadingMessages, error: messagesError } = useThreadMessages(
24+ selectedThreadId,
25+ optionalRoles,
26+ handleSetLoadedMessages
27+ );
28+
29+ return (
30+ <div className="relative flex-1 overflow-y-auto pb-48">
31+ {selectedThreadId ? (
32+ <MessageList
33+ messages={loadedMessages}
34+ isLoading={isLoadingMessages}
35+ error={messagesError ?? null}
36+ />
37+ ) : (
38+ <Welcome />
39+ )}
40+ <MessageEditor setLoadedMessages={setLoadedMessages} />
41+ </div>
42+ );
43+}
src/components/MessageEditor.tsx+48−14
@@ -7,11 +7,16 @@ import TrashIcon from "@/src/components/icons/TrashIcon";
77 import { useNewThread } from "@/src/hooks/api/useNewThread";
88 import { useNewThreadMessage } from "@/src/hooks/api/useNewThreadMessage";
99 import { useAutoResizeTextarea } from "@/src/hooks/useAutoResizeTextarea";
10-import { useState } from "react";
10+import { Message } from "@/src/lib/db/schema";
11+import { Dispatch, SetStateAction, useEffect, useState } from "react";
1112
12-export default function MessageEditor() {
13+export default function MessageEditor({
14+ setLoadedMessages,
15+}: {
16+ setLoadedMessages: Dispatch<SetStateAction<Message[]>>;
17+}) {
1318 const [userMessage, setUserMessage] = useState("");
14- const textareaRef = useAutoResizeTextarea(userMessage);
19+ const [isLoading, setIsLoading] = useState(false);
1520 const {
1621 selectedModel,
1722 models,
@@ -20,6 +25,39 @@ export default function MessageEditor() {
2025 selectedThreadId,
2126 setSelectedThreadId,
2227 } = useAppContext();
28+ const textareaRef = useAutoResizeTextarea(userMessage);
29+
30+ const { mutate: createThreadMessage, isPending: isCreatingThreadMessage } = useNewThreadMessage(
31+ selectedThreadId,
32+ userMessage,
33+ (msg) => {
34+ setLoadedMessages(
35+ (previousMessages) =>
36+ [
37+ ...previousMessages.filter((message) => message.id !== 0), // Remove messages that are not saved yet
38+ {
39+ id: 0, // Means it's not saved yet
40+ createdAt: new Date(),
41+ threadId: selectedThreadId,
42+ role: "user",
43+ content: userMessage,
44+ },
45+ {
46+ id: 0, // Means it's not saved yet
47+ createdAt: new Date(),
48+ threadId: selectedThreadId,
49+ role: msg.role,
50+ content: msg.content,
51+ thinking: msg.thinking,
52+ },
53+ ] as Message[]
54+ );
55+ },
56+ () => {
57+ setUserMessage("");
58+ textareaRef.current?.focus();
59+ }
60+ );
2361
2462 const { mutate: createThread, isPending: isCreatingThread } = useNewThread(
2563 userMessage,
@@ -29,20 +67,16 @@ export default function MessageEditor() {
2967 new Date().toISOString(),
3068 Intl.DateTimeFormat().resolvedOptions().timeZone,
3169 (threadId: number) => {
32- setUserMessage("");
33- setSelectedThreadId(threadId);
34- }
35- );
36-
37- const { mutate: createThreadMessage, isPending: isCreatingThreadMessage } = useNewThreadMessage(
38- selectedThreadId,
39- userMessage,
40- () => {
41- setUserMessage("");
70+ setSelectedThreadId(() => {
71+ createThreadMessage();
72+ return threadId;
73+ });
4274 }
4375 );
4476
45- const isLoading = isCreatingThread || isCreatingThreadMessage;
77+ useEffect(() => {
78+ setIsLoading(isCreatingThread || isCreatingThreadMessage);
79+ }, [isCreatingThread, isCreatingThreadMessage]);
4680
4781 const handleSave = () => {
4882 if (!selectedThreadId) {
src/components/MessageList.tsx+36−34
@@ -4,7 +4,6 @@ import { useAppContext } from "@/src/components/App";
44 import Button from "@/src/components/Button";
55 import TrashIcon from "@/src/components/icons/TrashIcon";
66 import useDeleteMessage from "@/src/hooks/api/useDeleteMessage";
7-import { useThreadMessages } from "@/src/hooks/api/useThreadMessages";
87 import { Message } from "@/src/lib/db/schema";
98 import { useEffect, useRef, useState } from "react";
109 import ReactMarkdown from "react-markdown";
@@ -86,63 +85,66 @@ function UserMessage({ message }: { message: Message }) {
8685 }
8786
8887 function AssistantMessage({ message }: { message: Message }) {
88+ const [isCollapsed, setIsCollapsed] = useState(message.id !== 0); // Only show opened optimistic thinking rendered messages
89+ const { showToolMessages } = useAppContext();
90+
8991 return (
9092 <BaseMessage message={message} justify="start" align="left">
91- <div className="prose prose-neutral max-w-none dark:prose-invert">
92- <ReactMarkdown remarkPlugins={[remarkGfm]} rehypePlugins={[rehypeHighlight]}>
93- {message.content}
94- </ReactMarkdown>
93+ <div className="flex flex-col gap-8">
94+ {showToolMessages && message.thinking && (
95+ <div
96+ onClick={() => setIsCollapsed(!isCollapsed)}
97+ className={`m-0 cursor-pointer overflow-hidden whitespace-pre-wrap break-words bg-transparent p-0 text-xs leading-normal text-neutral-500 ${isCollapsed ? "line-clamp-2" : ""}`}
98+ >
99+ <span className="font-bold">thinking:</span> {message.thinking}
100+ </div>
101+ )}
102+ <div className="prose prose-neutral max-w-none dark:prose-invert">
103+ <ReactMarkdown remarkPlugins={[remarkGfm]} rehypePlugins={[rehypeHighlight]}>
104+ {message.content}
105+ </ReactMarkdown>
106+ </div>
95107 </div>
96108 </BaseMessage>
97109 );
98110 }
99111
100-export default function MessageList() {
101- const { selectedThreadId, showSystemMessages, showToolMessages } = useAppContext();
102- const optionalRoles = [
103- ...(showSystemMessages ? ["system"] : []),
104- ...(showToolMessages ? ["tool"] : []),
105- ];
106- const {
107- data: messagesData,
108- isLoading: isLoadingMessages,
109- error: messagesError,
110- } = useThreadMessages(selectedThreadId, optionalRoles);
111-
112+export default function MessageList({
113+ messages,
114+ isLoading,
115+ error,
116+}: {
117+ messages: Message[];
118+ isLoading: boolean;
119+ error: Error | null;
120+}) {
112121 const listRef = useRef<HTMLDivElement | null>(null);
113- const prevMessageCount = useRef<number>(0);
114122
115123 useEffect(() => {
116- const currentMessageCount = messagesData?.messages.length ?? 0;
117- if (
118- listRef.current &&
119- currentMessageCount > prevMessageCount.current &&
120- currentMessageCount > 0
121- ) {
122- // Scroll the last message into view
124+ // Scroll to the last message on every render
125+ if (listRef.current && messages.length > 0) {
123126 const lastMessageElement = listRef.current.querySelector(".message:last-child");
124127 if (lastMessageElement) {
125128 (lastMessageElement as HTMLElement).scrollIntoView({ behavior: "smooth", block: "start" });
126129 }
127130 }
128- prevMessageCount.current = currentMessageCount;
129- }, [messagesData?.messages.length]);
131+ });
130132
131133 return (
132134 <div className="mx-auto max-w-5xl p-8">
133- {isLoadingMessages ? (
135+ {isLoading ? (
134136 <div>Loading messages...</div>
135- ) : messagesError ? (
136- <div>Error loading messages: {messagesError.message}</div>
137+ ) : error ? (
138+ <div>Error loading messages: {error.message}</div>
137139 ) : (
138140 <div ref={listRef} className="flex flex-col gap-8">
139- {messagesData?.messages.map((message) =>
141+ {messages.map((message) =>
140142 message.role === "system" || message.role === "tool" ? (
141- <SystemOrToolMessage key={message.id} message={message} />
143+ <SystemOrToolMessage key={message.id + message.role} message={message} />
142144 ) : message.role === "user" ? (
143- <UserMessage key={message.id} message={message} />
145+ <UserMessage key={message.id + message.role} message={message} />
144146 ) : (
145- <AssistantMessage key={message.id} message={message} />
147+ <AssistantMessage key={message.id + message.role} message={message} />
146148 )
147149 )}
148150 </div>
src/hooks/api/useNewThreadMessage.ts+37−4
@@ -1,16 +1,22 @@
11 "use client";
22
3-import { Message } from "@/src/lib/db/schema";
43 import { useMutation, useQueryClient } from "@tanstack/react-query";
54
5+type StreamedMessage = {
6+ role: string;
7+ content: string;
8+ thinking?: string;
9+};
10+
611 export function useNewThreadMessage(
712 threadId: number | null,
813 message: string,
14+ onMessageChunk?: (msg: StreamedMessage) => void,
915 onSuccess?: () => void
1016 ) {
1117 const queryClient = useQueryClient();
1218
13- return useMutation<{ message: Message }, Error>({
19+ return useMutation<void, Error>({
1420 mutationFn: async () => {
1521 if (!threadId) {
1622 throw new Error("Thread ID is required");
@@ -20,10 +26,37 @@ export function useNewThreadMessage(
2026 method: "POST",
2127 body: JSON.stringify({ message }),
2228 });
23- if (!response.ok) {
29+
30+ if (!response.ok || !response.body) {
2431 throw new Error(`Failed to create message: ${response.statusText}`);
2532 }
26- return response.json();
33+
34+ const reader = response.body.getReader();
35+ const decoder = new TextDecoder("utf-8");
36+ let accumulated = "";
37+
38+ while (true) {
39+ const { done, value } = await reader.read();
40+ if (done) {
41+ break;
42+ }
43+ accumulated += decoder.decode(value, { stream: true });
44+
45+ // Split stream into lines
46+ const lines = accumulated.split("\n\n");
47+ accumulated = lines.pop() || "";
48+
49+ for (const line of lines) {
50+ if (line.startsWith("data: ")) {
51+ try {
52+ const json = JSON.parse(line.replace(/^data: /, ""));
53+ onMessageChunk?.(json as StreamedMessage);
54+ } catch {
55+ // ignore JSON parse errors for invalid chunks
56+ }
57+ }
58+ }
59+ }
2760 },
2861 onSuccess: () => {
2962 if (!threadId) {
src/hooks/api/useThreadMessages.ts+15−2
@@ -1,5 +1,6 @@
11 import { Message } from "@/src/lib/db/schema";
22 import { useQuery } from "@tanstack/react-query";
3+import { useEffect } from "react";
34
45 /**
56 * Fetches messages for a given thread using react-query.
@@ -8,8 +9,12 @@ import { useQuery } from "@tanstack/react-query";
89 * @param optionalRoles - (Optional) Array of message roles (e.g., ["user", "assistant"]) to filter retrieved messages.
910 * @returns react-query's useQuery result containing messages and error state.
1011 */
11-export function useThreadMessages(threadId: number | null, optionalRoles: string[] = []) {
12- return useQuery<{ messages: Message[] }, Error>({
12+export function useThreadMessages(
13+ threadId: number | null,
14+ optionalRoles: string[] = [],
15+ onSuccess: (messages: Message[]) => void
16+) {
17+ const queryResult = useQuery<{ messages: Message[] }, Error>({
1318 queryKey: ["threadMessages", threadId],
1419 queryFn: async () => {
1520 if (!threadId) {
@@ -28,4 +33,12 @@ export function useThreadMessages(threadId: number | null, optionalRoles: string
2833 },
2934 enabled: threadId !== null,
3035 });
36+
37+ useEffect(() => {
38+ if (queryResult.data) {
39+ onSuccess(queryResult.data.messages);
40+ }
41+ }, [queryResult.data, onSuccess]);
42+
43+ return queryResult;
3144 }
src/lib/chat.ts+110−62
@@ -1,18 +1,105 @@
1-import { debug } from "@/src/lib/debug";
1+import { db } from "@/src/lib/db";
2+import { messages, threads } from "@/src/lib/db/schema";
23 import { createMessage } from "@/src/lib/messages";
3-import {
4- duckDuckGoTool,
5- executeToolCall,
6- fetchPageTool,
7- weatherTool,
8- webSearchTool,
9-} from "@/src/lib/tools";
4+import { executeToolCall, fetchPageTool, weatherTool, webSearchTool } from "@/src/lib/tools";
105 import { asc, eq } from "drizzle-orm";
11-import ollama, { type Tool } from "ollama";
12-import { db } from "./db";
13-import { messages, threads } from "./db/schema";
6+import ollama, { type Tool, type ToolCall } from "ollama";
147
15-export async function generateResponse(threadId: number) {
8+type MessageHistoryEntry = {
9+ role: "user" | "assistant" | "system" | "tool";
10+ content: string;
11+ thinking?: string;
12+};
13+
14+type MessageHistory = MessageHistoryEntry[];
15+
16+export function streamResponse(
17+ onChunk: (enqueue: (message: MessageHistoryEntry & { done?: boolean }) => void) => void
18+) {
19+ const encoder = new TextEncoder();
20+
21+ const stream = new ReadableStream({
22+ start(controller) {
23+ const enqueue = (message: MessageHistoryEntry & { done?: boolean }) => {
24+ const encoded = encoder.encode("data: " + JSON.stringify(message) + "\n\n");
25+ controller.enqueue(encoded);
26+ if (message.done) {
27+ controller.close();
28+ }
29+ };
30+
31+ onChunk(enqueue);
32+ },
33+ });
34+
35+ return stream;
36+}
37+
38+async function agentLoop(
39+ model: string,
40+ messageHistory: MessageHistory,
41+ enqueue: (message: MessageHistoryEntry & { done?: boolean }) => void
42+) {
43+ const tools: Tool[] = [weatherTool, fetchPageTool, webSearchTool];
44+ const newMessages: MessageHistory = [];
45+ let maxIterations = 6;
46+
47+ while (true) {
48+ maxIterations--;
49+
50+ const stream = await ollama.chat({
51+ model: model,
52+ messages: [...messageHistory, ...newMessages],
53+ tools: maxIterations > 0 ? tools : undefined,
54+ think: true,
55+ stream: true,
56+ });
57+
58+ let thinking = "";
59+ let content = "";
60+ const calls: ToolCall[] = [];
61+
62+ for await (const chunk of stream) {
63+ if (chunk.message.thinking) {
64+ thinking += chunk.message.thinking;
65+ enqueue({ role: "assistant", content: content, thinking: thinking });
66+ }
67+ if (chunk.message.content) {
68+ content += chunk.message.content;
69+ enqueue({ role: "assistant", content: content, thinking: thinking });
70+ }
71+ if (chunk.message.tool_calls?.length) {
72+ calls.push(...chunk.message.tool_calls);
73+ }
74+ }
75+
76+ if (content) {
77+ newMessages.push({ role: "assistant", content: content, thinking: thinking });
78+ }
79+ if (calls.length) {
80+ for (const call of calls) {
81+ const toolResponse = await executeToolCall(call);
82+ newMessages.push({ role: "tool", content: `[${call.function.name}] ${toolResponse}` });
83+ enqueue({ role: "tool", content: `[${call.function.name}] ${toolResponse}` });
84+ }
85+ } else {
86+ enqueue({
87+ role: "assistant",
88+ content: content,
89+ thinking: thinking,
90+ done: true,
91+ });
92+ break;
93+ }
94+ }
95+
96+ return newMessages;
97+}
98+
99+export async function generateResponse(
100+ threadId: number,
101+ enqueue: (message: MessageHistoryEntry) => void
102+) {
16103 if (!threadId) {
17104 throw new Error("Thread ID is required");
18105 }
@@ -24,63 +111,24 @@ export async function generateResponse(threadId: number) {
24111 }
25112
26113 // Get all of the messages for this thread:
27- const messagesList = await db
28- .select()
114+ const messagesList = (await db
115+ .select({
116+ role: messages.role,
117+ content: messages.content,
118+ thinking: messages.thinking,
119+ })
29120 .from(messages)
30121 .where(eq(messages.threadId, threadId))
31- .orderBy(asc(messages.createdAt));
122+ .orderBy(asc(messages.createdAt))) as MessageHistory;
32123
33124 if (messagesList.length === 0) {
34125 throw new Error("No messages found for this thread");
35126 }
36127
37- const tools: Tool[] = [duckDuckGoTool, weatherTool, fetchPageTool, webSearchTool];
38- const chatMessages = messagesList.map((message) => ({
39- role: message.role,
40- content: message.content,
41- }));
42- let iterations = 0;
43- let toolCallsExist = true;
44-
45- // Repeatedly generate model responses until no tool calls remain
46- while (toolCallsExist) {
47- iterations++;
48-
49- const response = await ollama.chat({
50- model: thread[0].model,
51- messages: chatMessages,
52- tools: iterations >= 10 ? undefined : tools, // Avoid looping forever
53- think: true,
54- });
55-
56- if (response.message.thinking) {
57- debug("Thinking:", response.message.thinking);
58- }
128+ const messageHistory = await agentLoop(thread[0].model, messagesList, enqueue);
59129
60- if (response.message?.tool_calls && response.message.tool_calls.length > 0) {
61- for (const toolCall of response.message.tool_calls) {
62- try {
63- const toolResponse = await executeToolCall(toolCall);
64- await createMessage(`${toolCall.function.name}: ${toolResponse}`, "tool", threadId);
65- chatMessages.push({
66- role: "tool",
67- content: toolResponse,
68- });
69- } catch (error) {
70- const content = `${toolCall.function.name}: Error: ${error instanceof Error ? error.message : "Unknown error"}`;
71- await createMessage(content, "tool", threadId);
72- chatMessages.push({
73- role: "tool",
74- content: content,
75- });
76-
77- debug(content);
78- }
79- }
80- } else {
81- toolCallsExist = false;
82- // insert the final response into the thread:
83- await createMessage(response.message.content, "assistant", threadId);
84- }
130+ // Update the thread with the new message history:
131+ for (const message of messageHistory) {
132+ await createMessage(message.content, message.role, threadId, message.thinking);
85133 }
86134 }
src/lib/db/schema.ts+2−5
@@ -5,7 +5,7 @@ import { integer, pgTable, serial, text, timestamp } from "drizzle-orm/pg-core";
55 export const threads = pgTable("threads", {
66 id: serial("id").primaryKey(),
77 title: text("title"),
8- model: text("model").notNull().default("gpt-oss"),
8+ model: text("model").notNull(),
99 maxPromptLength: integer("max_prompt_length"), // null, 1024, or 4096
1010 userPrompt: text("user_prompt"), // User-defined prompt to include in system prompt
1111 createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
@@ -19,11 +19,8 @@ export const messages = pgTable("messages", {
1919 .references(() => threads.id, { onDelete: "cascade" }),
2020 role: text("role").notNull(), // "user" or "assistant"
2121 content: text("content").notNull(),
22- model: text("model"), // Model used to generate this message (for assistant messages)
23- maxPromptLength: integer("max_prompt_length"), // Prompt size used to generate this message (for assistant messages) - null, 1024, or 4096
22+ thinking: text("thinking"), // Thinking (for assistant messages)
2423 createdAt: timestamp("created_at", { withTimezone: true }).notNull().defaultNow(),
25- generationTimeMs: integer("generation_time_ms"), // Time taken to generate response (for assistant messages)
26- toolCallCounts: text("tool_call_counts"), // JSON string of tool name counts, e.g. {"query_weather": 2, "query_web_search": 1}
2724 });
2825
2926 // Define relations
src/lib/messages.ts+3−1
@@ -4,12 +4,14 @@ import { messages } from "@/src/lib/db/schema";
44 export async function createMessage(
55 content: string,
66 role: "user" | "assistant" | "system" | "tool",
7- threadId: number
7+ threadId: number,
8+ thinking?: string
89 ) {
910 await db.insert(messages).values({
1011 threadId,
1112 role,
1213 content,
14+ thinking,
1315 createdAt: new Date(),
1416 });
1517 }
src/lib/threads.ts+0−5
@@ -1,4 +1,3 @@
1-import { generateResponse } from "@/src/lib/chat";
21 import { db } from "@/src/lib/db";
32 import { threads } from "@/src/lib/db/schema";
43 import { debug } from "@/src/lib/debug";
@@ -110,10 +109,6 @@ export async function createThread(
110109
111110 // Insert system message with guidelines
112111 await createMessage(systemPrompt, "system", threadId);
113- await createMessage(userMessage, "user", threadId);
114-
115- // Generate a response from the model.
116- await generateResponse(threadId);
117112
118113 return newThread[0];
119114 }
src/lib/tools/duckduckgo.ts+0−85
@@ -1,85 +0,0 @@
1-import { getCache, setCache } from "@/src/lib/cache";
2-import { createRateLimiter } from "@/src/lib/ratelimit";
3-
4-const CACHE_TTL_MS = 6 * 60 * 60 * 1000; // 6 hours
5-const duckDuckGoRateLimiter = createRateLimiter({
6- maxRequests: 500,
7- windowMs: 24 * 60 * 60 * 1000, // 24 hours in milliseconds
8- identifier: "duckduckgo",
9-});
10-
11-/**
12- * Queries DuckDuckGo Instant Answer API. Results cached 6h.
13- */
14-export async function queryDuckDuckGo(query: string): Promise<string> {
15- // Check cache first
16- const cacheKey = `duckduckgo:${query.toLowerCase().trim()}`;
17- const cached = await getCache<string>(cacheKey);
18- if (cached !== null) {
19- return cached;
20- }
21-
22- const rateLimitCheck = await duckDuckGoRateLimiter.check();
23- if (!rateLimitCheck.allowed) {
24- throw new Error("Service temporarily unavailable");
25- }
26-
27- await duckDuckGoRateLimiter.increment();
28- const url = "https://api.duckduckgo.com/";
29- const params = new URLSearchParams({
30- q: query,
31- format: "json",
32- no_html: "1",
33- skip_disambig: "1",
34- });
35-
36- const response = await fetch(`${url}?${params.toString()}`);
37- if (!response.ok) {
38- throw new Error("Service error");
39- }
40-
41- const data = await response.json();
42-
43- const parts: string[] = [];
44- if (data.Answer) parts.push(data.Answer);
45- if (data.AbstractText) parts.push(data.AbstractText);
46- if (data.Definition) parts.push(data.Definition);
47- if (data.Heading && !parts.length) parts.push(data.Heading);
48-
49- if (parts.length === 0) {
50- const result = `No answer found for: ${query}`;
51- await setCache(cacheKey, result, CACHE_TTL_MS);
52- return result;
53- }
54-
55- let result = parts[0];
56- if (data.AbstractURL) result += ` (${data.AbstractURL})`;
57- if (data.RelatedTopics?.length > 0) {
58- const topics = data.RelatedTopics.slice(0, 2)
59- .map((t: { Text?: string }) => t.Text)
60- .filter(Boolean)
61- .join(", ");
62- if (topics) result += ` | Related: ${topics}`;
63- }
64-
65- await setCache(cacheKey, result, CACHE_TTL_MS);
66- return result;
67-}
68-
69-export const duckDuckGoTool = {
70- type: "function" as const,
71- function: {
72- name: "query_duckduckgo",
73- description: "Get quick info from DuckDuckGo about topics, people, places, or concepts.",
74- parameters: {
75- type: "object",
76- properties: {
77- query: {
78- type: "string",
79- description: "Search query",
80- },
81- },
82- required: ["query"],
83- },
84- },
85-};
src/lib/tools/index.ts+0−4
@@ -1,12 +1,10 @@
11 /* personal-assistant-thing/src/lib/ollama/tools/index.ts */
22
3-import { queryDuckDuckGo } from "@/src/lib/tools/duckduckgo";
43 import { fetchPage } from "@/src/lib/tools/fetchpage";
54 import { queryWeather } from "@/src/lib/tools/weather";
65 import { queryWebSearch } from "@/src/lib/tools/websearch";
76 import type { ToolCall } from "ollama";
87
9-export { duckDuckGoTool, queryDuckDuckGo } from "@/src/lib/tools/duckduckgo";
108 export { fetchPage, fetchPageTool } from "@/src/lib/tools/fetchpage";
119 export { queryWeather, weatherTool } from "@/src/lib/tools/weather";
1210 export { queryWebSearch, webSearchTool } from "@/src/lib/tools/websearch";
@@ -23,8 +21,6 @@ export async function executeToolCall(toolCall: ToolCall) {
2321 return await fetchPage(toolCall.function.arguments.url);
2422 case "query_web_search":
2523 return await queryWebSearch(toolCall.function.arguments.query);
26- case "query_duckduckgo":
27- return await queryDuckDuckGo(toolCall.function.arguments.query);
2824 default:
2925 throw new Error(`Unknown tool: ${toolCall.function.name}`);
3026 }