xonecas.io
53129
Pa

PAT / commit bfb46b6

feat: stream agent responses

xonecas2025-12-17 bfb46b6d21bc1f99d09ee2e69e8c0681883f3bd1 parent 0514ecc

3 files changed

app/api/chat/route.ts+78−21
@@ -39,30 +39,87 @@ export async function POST(request: Request) {
3939 content: message,
4040 });
4141
42- // 4. Get assistant response
43- const {
44- content: answer,
45- generationTimeMs,
46- toolCalls,
47- } = await fetchOllamaResponse(ollamaMessages);
42+ // 4. Stream assistant response
43+ let accumulatedContent = "";
44+ let accumulatedThinking = "";
45+ const allToolCalls: any[] = [];
4846
49- // 5. Store assistant message with generation time and tool calls
50- await db.insert(messages).values({
51- threadId: parseInt(threadId),
52- role: "assistant",
53- content: answer,
54- createdAt: new Date(),
55- generationTimeMs,
56- toolCalls: toolCalls ? JSON.stringify(toolCalls) : null,
57- });
47+ const encoder = new TextEncoder();
48+ const stream = new ReadableStream({
49+ async start(controller) {
50+ try {
51+ const onChunk = (chunk: {
52+ content?: string;
53+ thinking?: string;
54+ toolCalls?: any[];
55+ }) => {
56+ if (chunk.content) {
57+ accumulatedContent += chunk.content;
58+ controller.enqueue(
59+ encoder.encode(`data: ${JSON.stringify({ type: "content", content: chunk.content })}\n\n`)
60+ );
61+ }
62+ if (chunk.thinking) {
63+ accumulatedThinking += chunk.thinking;
64+ controller.enqueue(
65+ encoder.encode(`data: ${JSON.stringify({ type: "thinking", thinking: chunk.thinking })}\n\n`)
66+ );
67+ }
68+ if (chunk.toolCalls) {
69+ allToolCalls.push(...chunk.toolCalls);
70+ controller.enqueue(
71+ encoder.encode(`data: ${JSON.stringify({ type: "toolCalls", toolCalls: chunk.toolCalls })}\n\n`)
72+ );
73+ }
74+ };
75+
76+ const { generationTimeMs, toolCalls } = await fetchOllamaResponse(
77+ ollamaMessages,
78+ onChunk
79+ );
5880
59- // 6. Update thread's updatedAt timestamp
60- await db
61- .update(threads)
62- .set({ updatedAt: new Date() })
63- .where(eq(threads.id, parseInt(threadId)));
81+ // 5. Store assistant message with generation time and tool calls
82+ await db.insert(messages).values({
83+ threadId: parseInt(threadId),
84+ role: "assistant",
85+ content: accumulatedContent,
86+ createdAt: new Date(),
87+ generationTimeMs,
88+ toolCalls: toolCalls ? JSON.stringify(toolCalls) : null,
89+ });
6490
65- return NextResponse.json({ answer });
91+ // 6. Update thread's updatedAt timestamp
92+ await db
93+ .update(threads)
94+ .set({ updatedAt: new Date() })
95+ .where(eq(threads.id, parseInt(threadId)));
96+
97+ // Send final message
98+ controller.enqueue(
99+ encoder.encode(
100+ `data: ${JSON.stringify({ type: "done", answer: accumulatedContent })}\n\n`
101+ )
102+ );
103+ controller.close();
104+ } catch (error) {
105+ console.error("Chat API error", error);
106+ controller.enqueue(
107+ encoder.encode(
108+ `data: ${JSON.stringify({ type: "error", error: "Sorry, something went wrong." })}\n\n`
109+ )
110+ );
111+ controller.close();
112+ }
113+ },
114+ });
115+
116+ return new Response(stream, {
117+ headers: {
118+ "Content-Type": "text/event-stream",
119+ "Cache-Control": "no-cache",
120+ Connection: "keep-alive",
121+ },
122+ });
66123 } catch (error) {
67124 console.error("Chat API error", error);
68125 return NextResponse.json({ answer: "Sorry, something went wrong." }, { status: 500 });
src/hooks/useMessages.ts+66−13
@@ -62,6 +62,19 @@ export function useMessages(threadId: number | null) {
6262 };
6363 setMessages((prev) => [...prev, userMsg]);
6464
65+ // Create assistant message placeholder
66+ const assistantMsgId = Date.now() + 1;
67+ const botMsg: Message = {
68+ id: assistantMsgId,
69+ threadId: targetThreadId,
70+ role: "assistant",
71+ content: "",
72+ createdAt: new Date(),
73+ generationTimeMs: null,
74+ toolCalls: null,
75+ };
76+ setMessages((prev) => [...prev, botMsg]);
77+
6578 try {
6679 const res = await fetch("/api/chat", {
6780 method: "POST",
@@ -69,26 +82,66 @@ export function useMessages(threadId: number | null) {
6982 body: JSON.stringify({ message: message.trim(), threadId: targetThreadId }),
7083 });
7184
72- const data = await res.json();
85+ if (!res.ok) {
86+ throw new Error(`HTTP error! status: ${res.status}`);
87+ }
88+
89+ if (!res.body) {
90+ throw new Error("Response body is null");
91+ }
7392
74- const botMsg: Message = {
75- id: Date.now() + 1,
76- threadId: targetThreadId,
77- role: "assistant",
78- content: data.answer || "",
79- createdAt: new Date(),
80- generationTimeMs: null,
81- toolCalls: null,
82- };
83- setMessages((prev) => [...prev, botMsg]);
93+ const reader = res.body.getReader();
94+ const decoder = new TextDecoder();
95+ let buffer = "";
96+ let accumulatedContent = "";
8497
85- // Reload messages to get correct IDs from database
86- await loadMessages(targetThreadId);
98+ while (true) {
99+ const { done, value } = await reader.read();
100+ if (done) break;
101+
102+ buffer += decoder.decode(value, { stream: true });
103+ const lines = buffer.split("\n");
104+ buffer = lines.pop() || ""; // Keep incomplete line in buffer
105+
106+ for (const line of lines) {
107+ if (line.trim() === "") continue; // Skip empty lines
108+ if (line.startsWith("data: ")) {
109+ try {
110+ const data = JSON.parse(line.slice(6));
111+ if (data.type === "content") {
112+ accumulatedContent += data.content || "";
113+ setMessages((prev) =>
114+ prev.map((msg) =>
115+ msg.id === assistantMsgId
116+ ? { ...msg, content: accumulatedContent }
117+ : msg
118+ )
119+ );
120+ } else if (data.type === "done") {
121+ // Final update with complete answer
122+ setMessages((prev) =>
123+ prev.map((msg) =>
124+ msg.id === assistantMsgId
125+ ? { ...msg, content: data.answer || accumulatedContent }
126+ : msg
127+ )
128+ );
129+ } else if (data.type === "error") {
130+ throw new Error(data.error || "Unknown error");
131+ }
132+ } catch (parseError) {
133+ console.error("Failed to parse SSE data", parseError);
134+ }
135+ }
136+ }
137+ }
87138
88139 // Reload threads to update the order
89140 onThreadsReload();
90141 } catch (error) {
91142 console.error("Failed to send message", error);
143+ // Remove the assistant message on error
144+ setMessages((prev) => prev.filter((msg) => msg.id !== assistantMsgId));
92145 } finally {
93146 setIsLoading(false);
94147 }
src/lib/ollama/index.ts+107−116
@@ -21,6 +21,12 @@ export interface OllamaResponse {
2121 toolCalls?: any[]; // Array of tool calls made during the conversation
2222 }
2323
24+export interface OllamaChunk {
25+ content?: string;
26+ thinking?: string;
27+ toolCalls?: any[];
28+}
29+
2430 /**
2531 * Executes a tool call and returns the result.
2632 *
@@ -28,8 +34,8 @@ export interface OllamaResponse {
2834 * @returns The tool response with the execution result.
2935 */
3036 async function executeToolCall(toolCall: {
31- id: string;
32- type: "function";
37+ id?: string;
38+ type?: "function";
3339 function: {
3440 name: string;
3541 arguments: string | { [key: string]: any };
@@ -40,57 +46,45 @@ async function executeToolCall(toolCall: {
4046
4147 debug(`[Tool] Executing: ${name}`, parsedArgs);
4248
49+ let result: string;
4350 switch (name) {
4451 case "query_duckduckgo":
45- const result = await queryDuckDuckGo(parsedArgs.query);
46- debug(`[Tool] ${name} completed, result length: ${result.length} chars`);
47- return {
48- tool_call_id: toolCall.id,
49- role: "tool",
50- name: "query_duckduckgo",
51- content: result,
52- };
52+ result = await queryDuckDuckGo(parsedArgs.query);
53+ break;
5354 case "query_weather":
54- const weatherResult = await queryWeather(parsedArgs.location);
55- debug(`[Tool] ${name} completed, result length: ${weatherResult.length} chars`);
56- return {
57- tool_call_id: toolCall.id,
58- role: "tool",
59- name: "query_weather",
60- content: weatherResult,
61- };
55+ result = await queryWeather(parsedArgs.location);
56+ break;
6257 case "query_web_search":
63- const webSearchResult = await queryWebSearch(parsedArgs.query);
64- debug(`[Tool] ${name} completed, result length: ${webSearchResult.length} chars`);
65- return {
66- tool_call_id: toolCall.id,
67- role: "tool",
68- name: "query_web_search",
69- content: webSearchResult,
70- };
58+ result = await queryWebSearch(parsedArgs.query);
59+ break;
7160 default:
7261 debug(`[Tool] Unknown tool: ${name}`);
73- return {
74- tool_call_id: toolCall.id,
75- role: "tool",
76- name,
77- content: `Error: Unknown tool "${name}"`,
78- };
62+ result = `Error: Unknown tool "${name}"`;
7963 }
64+
65+ debug(`[Tool] ${name} completed, result length: ${result.length} chars`);
66+ return {
67+ role: "tool",
68+ tool_name: name,
69+ content: result,
70+ };
8071 }
8172
8273 /**
8374 * Sends messages to the Ollama model using the chat API and returns the generated response.
84- * Supports tool calling - if the model requests tools, they are executed and the results
85- * are sent back to the model for a final response.
75+ * Supports tool calling with an agent loop - if the model requests tools, they are executed
76+ * and the results are sent back to the model until a final response is generated.
77+ * Uses streaming to process responses incrementally and can stream chunks to the UI.
8678 *
8779 * @param messages - Array of messages in the conversation history.
80+ * @param onChunk - Optional callback function to receive streaming chunks (content, thinking, toolCalls).
8881 * @param model - The Ollama model to use. Defaults to 'gpt-oss'.
8982 * @returns Object containing the response content and generation time in milliseconds.
9083 * @throws If the request fails.
9184 */
9285 export async function fetchOllamaResponse(
9386 messages: OllamaMessageInput[],
87+ onChunk?: (chunk: OllamaChunk) => void,
9488 model = "gpt-oss"
9589 ): Promise<OllamaResponse> {
9690 const tools = [duckDuckGoTool, weatherTool, webSearchTool];
@@ -99,111 +93,108 @@ export async function fetchOllamaResponse(
9993 role: msg.role,
10094 content: msg.content,
10195 }));
102- const maxIterations = 6; // Prevent infinite loops
103- let iterations = 0;
104- const allToolCalls: any[] = []; // Collect all tool calls across iterations
96+ const allToolCalls: any[] = [];
10597
10698 debug(`[Ollama] Starting chat with model: ${model}, messages: ${messages.length}`);
10799
108- // Exhaust tool calls: Keep iterating until the model stops requesting tools
109- // and returns a final response. Each iteration executes any requested tools
110- // and sends the results back to the model for processing.
111- while (iterations < maxIterations) {
112- iterations++;
113- debug(`[Ollama] Iteration ${iterations}/${maxIterations}, sending to model...`);
114-
115- const result = await ollama.chat({
100+ // Agent loop: Keep iterating until the model stops requesting tools
101+ while (true) {
102+ const stream = await ollama.chat({
116103 model,
117104 messages: currentMessages,
118105 tools,
106+ stream: true,
119107 });
120108
121- totalDuration += result.total_duration;
122- const iterationDuration = Math.round(result.total_duration / 1_000_000);
109+ let content = "";
110+ let thinking = "";
111+ const toolCalls: any[] = [];
112+ let iterationDuration = 0;
113+
114+ // Accumulate partial fields from streaming chunks
115+ for await (const chunk of stream) {
116+ if (chunk.total_duration) {
117+ iterationDuration = chunk.total_duration;
118+ }
119+
120+ if (chunk.message?.thinking) {
121+ thinking += chunk.message.thinking;
122+ onChunk?.({ thinking: chunk.message.thinking });
123+ }
124+
125+ if (chunk.message?.content) {
126+ content += chunk.message.content;
127+ onChunk?.({ content: chunk.message.content });
128+ }
129+
130+ // Accumulate tool calls from chunks
131+ if (chunk.message?.tool_calls?.length) {
132+ for (const toolCall of chunk.message.tool_calls) {
133+ // If tool call has an ID, check if we already have it
134+ const toolCallAny = toolCall as any;
135+ if (toolCallAny.id) {
136+ const existingIndex = toolCalls.findIndex((tc: any) => tc.id === toolCallAny.id);
137+ if (existingIndex >= 0) {
138+ toolCalls[existingIndex] = toolCall;
139+ } else {
140+ toolCalls.push(toolCall);
141+ }
142+ } else {
143+ toolCalls.push(toolCall);
144+ }
145+ }
146+ onChunk?.({ toolCalls: chunk.message.tool_calls });
147+ }
148+ }
149+
150+ totalDuration += iterationDuration;
151+ const iterationDurationMs = Math.round(iterationDuration / 1_000_000);
123152
124153 debug(
125- `[Ollama] Response received (${iterationDuration}ms), tool_calls: ${result.message.tool_calls?.length || 0}`
154+ `[Ollama] Response received (${iterationDurationMs}ms), tool_calls: ${toolCalls.length}`
126155 );
127156
128- // If the model made tool calls, execute them and continue the conversation
129- if (result.message.tool_calls && result.message.tool_calls.length > 0) {
130- debug(
131- `[Ollama] Tool calls detected:`,
132- result.message.tool_calls.map((tc: any) => ({
133- name: tc.function.name,
134- id: tc.id,
135- }))
136- );
137-
138- // Collect tool calls for the final response
139- allToolCalls.push(...result.message.tool_calls);
140-
141- // Add the assistant's message with tool calls (use result.message.tool_calls directly)
157+ // Append accumulated fields to messages
158+ if (thinking || content || toolCalls.length) {
142159 currentMessages.push({
143160 role: "assistant",
144- content: result.message.content || "",
145- tool_calls: result.message.tool_calls,
161+ thinking: thinking || undefined,
162+ content: content || "",
163+ tool_calls: toolCalls.length > 0 ? toolCalls : undefined,
146164 });
165+ }
147166
148- // Execute all tool calls
149- const toolResponses = await Promise.all(
150- result.message.tool_calls.map((toolCall: any) =>
151- executeToolCall({
152- id: toolCall.id || toolCall.function.name,
153- type: "function",
154- function: {
155- name: toolCall.function.name,
156- arguments:
157- typeof toolCall.function.arguments === "string"
158- ? toolCall.function.arguments
159- : toolCall.function.arguments,
160- },
161- })
162- )
163- );
167+ // If no tool calls, break the loop and return the final response
168+ if (!toolCalls.length) {
169+ const generationTimeMs = Math.round(totalDuration / 1_000_000);
164170
165- debug(`[Ollama] Tool responses received, continuing conversation...`);
171+ debug(
172+ `[Ollama] Final response (${generationTimeMs}ms total), content length: ${content.length} chars`
173+ );
166174
167- // Add tool responses to the conversation
168- currentMessages.push(...toolResponses);
169- continue;
175+ return {
176+ content,
177+ generationTimeMs,
178+ toolCalls: allToolCalls.length > 0 ? allToolCalls : undefined,
179+ };
170180 }
171181
172- // No tool calls, return the final response
173- const content = result.message.content;
174- const generationTimeMs = Math.round(totalDuration / 1_000_000); // Convert nanoseconds to milliseconds
175-
182+ // Execute tool calls and add results to messages
176183 debug(
177- `[Ollama] Final response (${generationTimeMs}ms total, ${iterations} iteration${iterations !== 1 ? "s" : ""}), content length: ${content.length} chars`
184+ `[Ollama] Tool calls detected:`,
185+ toolCalls.map((tc: any) => ({
186+ name: tc.function.name,
187+ id: tc.id,
188+ }))
178189 );
179190
180- return {
181- content,
182- generationTimeMs,
183- toolCalls: allToolCalls.length > 0 ? allToolCalls : undefined,
184- };
185- }
186-
187- // If we hit max iterations, send one more request without tools
188- debug(`[Ollama] Max iterations reached (${maxIterations}), sending final request without tools...`);
189-
190- const lastResult = await ollama.chat({
191- model,
192- messages: currentMessages,
193- // Don't pass tools to force a text-only response
194- });
195- totalDuration += lastResult.total_duration;
196-
197- const content = lastResult.message.content || "";
198- const generationTimeMs = Math.round(totalDuration / 1_000_000);
191+ allToolCalls.push(...toolCalls);
199192
200- debug(
201- `[Ollama] Final response after max iterations (${generationTimeMs}ms total, ${iterations} iterations), content length: ${content.length} chars`
202- );
193+ for (const toolCall of toolCalls) {
194+ const toolResponse = await executeToolCall(toolCall);
195+ currentMessages.push(toolResponse);
196+ }
203197
204- return {
205- content,
206- generationTimeMs,
207- toolCalls: allToolCalls.length > 0 ? allToolCalls : undefined,
208- };
198+ debug(`[Ollama] Tool responses received, continuing conversation...`);
199+ }
209200 }