xonecas.io
53129
Pa

PAT / commit 4c7d0e0

feat: steam agent thinking and response to the UI

xonecas2025-12-22 4c7d0e091f5917575fe7749e7ed1b021a45a8adc parent 80048f4

10 files changed

app/api/threads/[id]/messages/route.ts+13−4
@@ -1,4 +1,4 @@
1-import { generateResponse } from "@/src/lib/chat";
1+import { generateResponse, streamResponse } from "@/src/lib/chat";
22 import { db } from "@/src/lib/db";
33 import { messages } from "@/src/lib/db/schema";
44 import { debug } from "@/src/lib/debug";
@@ -63,10 +63,19 @@ export async function POST(request: Request, { params }: { params: Promise<{ id:
6363
6464 await createMessage(message, "user", threadId);
6565
66+ const stream = streamResponse(async (enqueue) => {
67+ if (threadId) {
68+ await generateResponse(threadId, enqueue);
69+ }
70+ });
6671 // Generate a response from the model.
67- await generateResponse(threadId);
68-
69- return NextResponse.json({ message: message });
72+ return new Response(stream, {
73+ headers: {
74+ "Content-Type": "text/event-stream",
75+ "Cache-Control": "no-cache",
76+ Connection: "keep-alive",
77+ },
78+ });
7079 } catch (err) {
7180 debug(
7281 `[Messages API] Error creating message for thread ${threadId ?? "unknown"}:`,
src/components/App.tsx+9−14
@@ -1,30 +1,28 @@
11 "use client";
22
3-import MessageEditor from "@/src/components/MessageEditor";
4-import MessageList from "@/src/components/MessageList";
3+import Main from "@/src/components/Main";
54 import Sidebar from "@/src/components/Sidebar";
6-import Welcome from "@/src/components/Welcome";
75 import { useModels, type Model } from "@/src/hooks/api/useModels";
86 import { useLocalStorage } from "@/src/hooks/useLocalStorage";
97 import { useTheme } from "@/src/hooks/useTheme";
10-import { createContext, useContext, useState } from "react";
8+import { createContext, Dispatch, SetStateAction, useContext, useState } from "react";
119
1210 export type AppContextType = {
1311 selectedModel: Model | null;
14- setSelectedModel: (model: Model) => void;
12+ setSelectedModel: Dispatch<SetStateAction<Model | null>>;
1513 maxPromptLength: number | null;
16- setMaxPromptLength: (length: number | null) => void;
14+ setMaxPromptLength: Dispatch<SetStateAction<number | null>>;
1715 userPrompt: string;
18- setUserPrompt: (prompt: string) => void;
16+ setUserPrompt: Dispatch<SetStateAction<string>>;
1917 models: Model[];
2018 isModelsLoading: boolean;
2119 modelsError: Error | null;
2220 selectedThreadId: number | null;
23- setSelectedThreadId: (threadId: number | null) => void;
21+ setSelectedThreadId: Dispatch<SetStateAction<number | null>>;
2422 showSystemMessages: boolean;
25- setShowSystemMessages: (show: boolean) => void;
23+ setShowSystemMessages: Dispatch<SetStateAction<boolean>>;
2624 showToolMessages: boolean;
27- setShowToolMessages: (show: boolean) => void;
25+ setShowToolMessages: Dispatch<SetStateAction<boolean>>;
2826 themeMode: "device" | "dark" | "light";
2927 handleThemeChange: (mode: "device" | "dark" | "light") => void;
3028 };
@@ -85,10 +83,7 @@ export default function App() {
8583 }}
8684 >
8785 <div className="flex h-screen overflow-hidden bg-neutral-100 text-neutral-800 dark:bg-neutral-900 dark:text-neutral-200">
88- <div className="relative flex-1 overflow-y-auto pb-48">
89- {selectedThreadId ? <MessageList /> : <Welcome />}
90- <MessageEditor />
91- </div>
86+ <Main />
9287 <Sidebar />
9388 </div>
9489 </AppContext.Provider>
src/components/Main.tsx+46−0
@@ -0,0 +1,46 @@
1+"use client";
2+
3+import { useAppContext } from "@/src/components/App";
4+import MessageEditor from "@/src/components/MessageEditor";
5+import MessageList from "@/src/components/MessageList";
6+import Welcome from "@/src/components/Welcome";
7+import { useThreadMessages } from "@/src/hooks/api/useThreadMessages";
8+import { Message } from "@/src/lib/db/schema";
9+import { useCallback, useState } from "react";
10+
11+export default function Main() {
12+ const [loadedMessages, setLoadedMessages] = useState<Message[]>([]);
13+ const { selectedThreadId, showSystemMessages, showToolMessages } = useAppContext();
14+ const optionalRoles = [
15+ ...(showSystemMessages ? ["system"] : []),
16+ ...(showToolMessages ? ["tool"] : []),
17+ ];
18+
19+ const handleSetLoadedMessages = useCallback((messages: Message[]) => {
20+ setLoadedMessages((previousMessages) => [
21+ ...messages,
22+ ...previousMessages.filter((message) => message.threadId === 0), // Keep messages that aren't saved yet
23+ ]);
24+ }, []);
25+
26+ const { isLoading: isLoadingMessages, error: messagesError } = useThreadMessages(
27+ selectedThreadId,
28+ optionalRoles,
29+ handleSetLoadedMessages
30+ );
31+
32+ return (
33+ <div className="relative flex-1 overflow-y-auto pb-48">
34+ {selectedThreadId ? (
35+ <MessageList
36+ messages={loadedMessages}
37+ isLoading={isLoadingMessages}
38+ error={messagesError ?? null}
39+ />
40+ ) : (
41+ <Welcome />
42+ )}
43+ <MessageEditor setLoadedMessages={setLoadedMessages} />
44+ </div>
45+ );
46+}
src/components/MessageEditor.tsx+41−16
@@ -7,11 +7,16 @@ import TrashIcon from "@/src/components/icons/TrashIcon";
77 import { useNewThread } from "@/src/hooks/api/useNewThread";
88 import { useNewThreadMessage } from "@/src/hooks/api/useNewThreadMessage";
99 import { useAutoResizeTextarea } from "@/src/hooks/useAutoResizeTextarea";
10-import { useState } from "react";
10+import { Message } from "@/src/lib/db/schema";
11+import { Dispatch, SetStateAction, useEffect, useState } from "react";
1112
12-export default function MessageEditor() {
13+export default function MessageEditor({
14+ setLoadedMessages,
15+}: {
16+ setLoadedMessages: Dispatch<SetStateAction<Message[]>>;
17+}) {
1318 const [userMessage, setUserMessage] = useState("");
14- const textareaRef = useAutoResizeTextarea(userMessage);
19+ const [isLoading, setIsLoading] = useState(false);
1520 const {
1621 selectedModel,
1722 models,
@@ -20,6 +25,32 @@ export default function MessageEditor() {
2025 selectedThreadId,
2126 setSelectedThreadId,
2227 } = useAppContext();
28+ const textareaRef = useAutoResizeTextarea(userMessage);
29+
30+ const { mutate: createThreadMessage, isPending: isCreatingThreadMessage } = useNewThreadMessage(
31+ selectedThreadId,
32+ userMessage,
33+ (msg) => {
34+ setLoadedMessages(
35+ (previousMessages) =>
36+ [
37+ ...previousMessages.filter((message) => message.id !== 0), // Remove messages that are not saved yet
38+ {
39+ id: 0, // Means it's not saved yet
40+ createdAt: new Date(),
41+ threadId: selectedThreadId,
42+ role: msg.role,
43+ content: msg.content,
44+ thinking: msg.thinking,
45+ },
46+ ] as Message[]
47+ );
48+ },
49+ () => {
50+ setUserMessage("");
51+ textareaRef.current?.focus();
52+ }
53+ );
2354
2455 const { mutate: createThread, isPending: isCreatingThread } = useNewThread(
2556 userMessage,
@@ -29,22 +60,16 @@ export default function MessageEditor() {
2960 new Date().toISOString(),
3061 Intl.DateTimeFormat().resolvedOptions().timeZone,
3162 (threadId: number) => {
32- setUserMessage("");
33- setSelectedThreadId(threadId);
34- textareaRef.current?.focus();
35- }
36- );
37-
38- const { mutate: createThreadMessage, isPending: isCreatingThreadMessage } = useNewThreadMessage(
39- selectedThreadId,
40- userMessage,
41- () => {
42- setUserMessage("");
43- textareaRef.current?.focus();
63+ setSelectedThreadId(() => {
64+ createThreadMessage();
65+ return threadId;
66+ });
4467 }
4568 );
4669
47- const isLoading = isCreatingThread || isCreatingThreadMessage;
70+ useEffect(() => {
71+ setIsLoading(isCreatingThread || isCreatingThreadMessage);
72+ }, [isCreatingThread, isCreatingThreadMessage]);
4873
4974 const handleSave = () => {
5075 if (!selectedThreadId) {
src/components/MessageList.tsx+26−24
@@ -1,10 +1,8 @@
11 "use client";
22
3-import { useAppContext } from "@/src/components/App";
43 import Button from "@/src/components/Button";
54 import TrashIcon from "@/src/components/icons/TrashIcon";
65 import useDeleteMessage from "@/src/hooks/api/useDeleteMessage";
7-import { useThreadMessages } from "@/src/hooks/api/useThreadMessages";
86 import { Message } from "@/src/lib/db/schema";
97 import { useEffect, useRef, useState } from "react";
108 import ReactMarkdown from "react-markdown";
@@ -88,32 +86,36 @@ function UserMessage({ message }: { message: Message }) {
8886 function AssistantMessage({ message }: { message: Message }) {
8987 return (
9088 <BaseMessage message={message} justify="start" align="left">
91- <div className="prose prose-neutral max-w-none dark:prose-invert">
92- <ReactMarkdown remarkPlugins={[remarkGfm]} rehypePlugins={[rehypeHighlight]}>
93- {message.content}
94- </ReactMarkdown>
89+ <div className="flex flex-col gap-4">
90+ {message.thinking && (
91+ <div className="prose prose-neutral max-w-none text-xs text-neutral-400 dark:prose-invert dark:text-neutral-600">
92+ <span className="font-bold">thinking:</span> {message.thinking}
93+ </div>
94+ )}
95+ <div className="prose prose-neutral max-w-none dark:prose-invert">
96+ <ReactMarkdown remarkPlugins={[remarkGfm]} rehypePlugins={[rehypeHighlight]}>
97+ {message.content}
98+ </ReactMarkdown>
99+ </div>
95100 </div>
96101 </BaseMessage>
97102 );
98103 }
99104
100-export default function MessageList() {
101- const { selectedThreadId, showSystemMessages, showToolMessages } = useAppContext();
102- const optionalRoles = [
103- ...(showSystemMessages ? ["system"] : []),
104- ...(showToolMessages ? ["tool"] : []),
105- ];
106- const {
107- data: messagesData,
108- isLoading: isLoadingMessages,
109- error: messagesError,
110- } = useThreadMessages(selectedThreadId, optionalRoles);
111-
105+export default function MessageList({
106+ messages,
107+ isLoading,
108+ error,
109+}: {
110+ messages: Message[];
111+ isLoading: boolean;
112+ error: Error | null;
113+}) {
112114 const listRef = useRef<HTMLDivElement | null>(null);
113115 const prevMessageCount = useRef<number>(0);
114116
115117 useEffect(() => {
116- const currentMessageCount = messagesData?.messages.length ?? 0;
118+ const currentMessageCount = messages.length;
117119 if (
118120 listRef.current &&
119121 currentMessageCount > prevMessageCount.current &&
@@ -126,17 +128,17 @@ export default function MessageList() {
126128 }
127129 }
128130 prevMessageCount.current = currentMessageCount;
129- }, [messagesData?.messages.length]);
131+ }, [messages.length]);
130132
131133 return (
132134 <div className="mx-auto max-w-5xl p-8">
133- {isLoadingMessages ? (
135+ {isLoading ? (
134136 <div>Loading messages...</div>
135- ) : messagesError ? (
136- <div>Error loading messages: {messagesError.message}</div>
137+ ) : error ? (
138+ <div>Error loading messages: {error.message}</div>
137139 ) : (
138140 <div ref={listRef} className="flex flex-col gap-8">
139- {messagesData?.messages.map((message) =>
141+ {messages.map((message) =>
140142 message.role === "system" || message.role === "tool" ? (
141143 <SystemOrToolMessage key={message.id} message={message} />
142144 ) : message.role === "user" ? (
src/hooks/api/useNewThreadMessage.ts+38−4
@@ -1,16 +1,22 @@
11 "use client";
22
3-import { Message } from "@/src/lib/db/schema";
43 import { useMutation, useQueryClient } from "@tanstack/react-query";
54
5+type StreamedMessage = {
6+ role: string;
7+ content: string;
8+ thinking?: string;
9+};
10+
611 export function useNewThreadMessage(
712 threadId: number | null,
813 message: string,
14+ onMessageChunk?: (msg: StreamedMessage) => void,
915 onSuccess?: () => void
1016 ) {
1117 const queryClient = useQueryClient();
1218
13- return useMutation<{ message: Message }, Error>({
19+ return useMutation<void, Error>({
1420 mutationFn: async () => {
1521 if (!threadId) {
1622 throw new Error("Thread ID is required");
@@ -20,10 +26,38 @@ export function useNewThreadMessage(
2026 method: "POST",
2127 body: JSON.stringify({ message }),
2228 });
23- if (!response.ok) {
29+
30+ if (!response.ok || !response.body) {
2431 throw new Error(`Failed to create message: ${response.statusText}`);
2532 }
26- return response.json();
33+
34+ const reader = response.body.getReader();
35+ const decoder = new TextDecoder("utf-8");
36+ let accumulated = "";
37+
38+ while (true) {
39+ const { done, value } = await reader.read();
40+ if (done) {
41+ console.log("[useNewThreadMessage] Stream finished reading");
42+ break;
43+ }
44+ accumulated += decoder.decode(value, { stream: true });
45+
46+ // Split stream into lines
47+ const lines = accumulated.split("\n\n");
48+ accumulated = lines.pop() || "";
49+
50+ for (const line of lines) {
51+ if (line.startsWith("data: ")) {
52+ try {
53+ const json = JSON.parse(line.replace(/^data: /, ""));
54+ onMessageChunk?.(json as StreamedMessage);
55+ } catch {
56+ // ignore JSON parse errors for invalid chunks
57+ }
58+ }
59+ }
60+ }
2761 },
2862 onSuccess: () => {
2963 if (!threadId) {
src/hooks/api/useThreadMessages.ts+15−2
@@ -1,5 +1,6 @@
11 import { Message } from "@/src/lib/db/schema";
22 import { useQuery } from "@tanstack/react-query";
3+import { useEffect } from "react";
34
45 /**
56 * Fetches messages for a given thread using react-query.
@@ -8,8 +9,12 @@ import { useQuery } from "@tanstack/react-query";
89 * @param optionalRoles - (Optional) Array of message roles (e.g., ["user", "assistant"]) to filter retrieved messages.
910 * @returns react-query's useQuery result containing messages and error state.
1011 */
11-export function useThreadMessages(threadId: number | null, optionalRoles: string[] = []) {
12- return useQuery<{ messages: Message[] }, Error>({
12+export function useThreadMessages(
13+ threadId: number | null,
14+ optionalRoles: string[] = [],
15+ onSuccess: (messages: Message[]) => void
16+) {
17+ const queryResult = useQuery<{ messages: Message[] }, Error>({
1318 queryKey: ["threadMessages", threadId],
1419 queryFn: async () => {
1520 if (!threadId) {
@@ -28,4 +33,12 @@ export function useThreadMessages(threadId: number | null, optionalRoles: string
2833 },
2934 enabled: threadId !== null,
3035 });
36+
37+ useEffect(() => {
38+ if (queryResult.data) {
39+ onSuccess(queryResult.data.messages);
40+ }
41+ }, [queryResult.data, onSuccess]);
42+
43+ return queryResult;
3144 }
src/lib/chat.ts+47−7
@@ -5,13 +5,41 @@ import { executeToolCall, fetchPageTool, weatherTool, webSearchTool } from "@/sr
55 import { asc, eq } from "drizzle-orm";
66 import ollama, { type Tool, type ToolCall } from "ollama";
77
8-type MessageHistory = {
8+type MessageHistoryEntry = {
99 role: "user" | "assistant" | "system" | "tool";
1010 content: string;
1111 thinking?: string;
12-}[];
12+};
1313
14-async function agentLoop(model: string, messageHistory: MessageHistory) {
14+type MessageHistory = MessageHistoryEntry[];
15+
16+export function streamResponse(
17+ onChunk: (enqueue: (message: MessageHistoryEntry & { done?: boolean }) => void) => void
18+) {
19+ const encoder = new TextEncoder();
20+
21+ const stream = new ReadableStream({
22+ start(controller) {
23+ const enqueue = (message: MessageHistoryEntry & { done?: boolean }) => {
24+ const encoded = encoder.encode("data: " + JSON.stringify(message) + "\n\n");
25+ controller.enqueue(encoded);
26+ if (message.done) {
27+ controller.close();
28+ }
29+ };
30+
31+ onChunk(enqueue);
32+ },
33+ });
34+
35+ return stream;
36+}
37+
38+async function agentLoop(
39+ model: string,
40+ messageHistory: MessageHistory,
41+ enqueue: (message: MessageHistoryEntry & { done?: boolean }) => void
42+) {
1543 const tools: Tool[] = [weatherTool, fetchPageTool, webSearchTool];
1644 const newMessages: MessageHistory = [];
1745 let maxIterations = 6;
@@ -33,10 +61,12 @@ async function agentLoop(model: string, messageHistory: MessageHistory) {
3361
3462 for await (const chunk of stream) {
3563 if (chunk.message.thinking) {
36- thinking += chunk.message;
64+ thinking += chunk.message.thinking;
65+ enqueue({ role: "assistant", content: content, thinking: thinking });
3766 }
3867 if (chunk.message.content) {
3968 content += chunk.message.content;
69+ enqueue({ role: "assistant", content: content, thinking: thinking });
4070 }
4171 if (chunk.message.tool_calls?.length) {
4272 calls.push(...chunk.message.tool_calls);
@@ -50,8 +80,15 @@ async function agentLoop(model: string, messageHistory: MessageHistory) {
5080 for (const call of calls) {
5181 const toolResponse = await executeToolCall(call);
5282 newMessages.push({ role: "tool", content: `[${call.function.name}] ${toolResponse}` });
83+ enqueue({ role: "tool", content: `[${call.function.name}] ${toolResponse}` });
5384 }
5485 } else {
86+ enqueue({
87+ role: "assistant",
88+ content: content,
89+ thinking: thinking,
90+ done: true,
91+ });
5592 break;
5693 }
5794 }
@@ -59,7 +96,10 @@ async function agentLoop(model: string, messageHistory: MessageHistory) {
5996 return newMessages;
6097 }
6198
62-export async function generateResponse(threadId: number) {
99+export async function generateResponse(
100+ threadId: number,
101+ enqueue: (message: MessageHistoryEntry) => void
102+) {
63103 if (!threadId) {
64104 throw new Error("Thread ID is required");
65105 }
@@ -85,10 +125,10 @@ export async function generateResponse(threadId: number) {
85125 throw new Error("No messages found for this thread");
86126 }
87127
88- const messageHistory = await agentLoop(thread[0].model, messagesList);
128+ const messageHistory = await agentLoop(thread[0].model, messagesList, enqueue);
89129
90130 // Update the thread with the new message history:
91131 for (const message of messageHistory) {
92- await createMessage(message.content, message.role, threadId);
132+ await createMessage(message.content, message.role, threadId, message.thinking);
93133 }
94134 }
src/lib/messages.ts+3−1
@@ -4,12 +4,14 @@ import { messages } from "@/src/lib/db/schema";
44 export async function createMessage(
55 content: string,
66 role: "user" | "assistant" | "system" | "tool",
7- threadId: number
7+ threadId: number,
8+ thinking?: string
89 ) {
910 await db.insert(messages).values({
1011 threadId,
1112 role,
1213 content,
14+ thinking,
1315 createdAt: new Date(),
1416 });
1517 }
src/lib/threads.ts+0−5
@@ -1,4 +1,3 @@
1-import { generateResponse } from "@/src/lib/chat";
21 import { db } from "@/src/lib/db";
32 import { threads } from "@/src/lib/db/schema";
43 import { debug } from "@/src/lib/debug";
@@ -110,10 +109,6 @@ export async function createThread(
110109
111110 // Insert system message with guidelines
112111 await createMessage(systemPrompt, "system", threadId);
113- await createMessage(userMessage, "user", threadId);
114-
115- // Generate a response from the model.
116- await generateResponse(threadId);
117112
118113 return newThread[0];
119114 }