xonecas.io
53129
Pa

PAT / commit 969c758

feat: optimize token use from tools

xonecas2025-12-18 969c758bd7755fd0acfec65546c618997e05b92b parent e0a3c6b

5 files changed

drizzle/0005_harsh_wild_pack.sql+1−0
@@ -0,0 +1 @@
1+ALTER TABLE "messages" RENAME COLUMN "tool_calls" TO "tool_call_counts";
drizzle/meta/0005_snapshot.json+153−0
@@ -0,0 +1,153 @@
1+{
2+ "id": "c986ec80-6630-4a52-97f2-3df6e02ca96b",
3+ "prevId": "bc4cb2c1-be83-4071-9f0d-1acb6b257269",
4+ "version": "7",
5+ "dialect": "postgresql",
6+ "tables": {
7+ "public.messages": {
8+ "name": "messages",
9+ "schema": "",
10+ "columns": {
11+ "id": {
12+ "name": "id",
13+ "type": "serial",
14+ "primaryKey": true,
15+ "notNull": true
16+ },
17+ "thread_id": {
18+ "name": "thread_id",
19+ "type": "integer",
20+ "primaryKey": false,
21+ "notNull": true
22+ },
23+ "role": {
24+ "name": "role",
25+ "type": "text",
26+ "primaryKey": false,
27+ "notNull": true
28+ },
29+ "content": {
30+ "name": "content",
31+ "type": "text",
32+ "primaryKey": false,
33+ "notNull": true
34+ },
35+ "model": {
36+ "name": "model",
37+ "type": "text",
38+ "primaryKey": false,
39+ "notNull": false
40+ },
41+ "max_prompt_length": {
42+ "name": "max_prompt_length",
43+ "type": "integer",
44+ "primaryKey": false,
45+ "notNull": false
46+ },
47+ "created_at": {
48+ "name": "created_at",
49+ "type": "timestamp with time zone",
50+ "primaryKey": false,
51+ "notNull": true,
52+ "default": "now()"
53+ },
54+ "generation_time_ms": {
55+ "name": "generation_time_ms",
56+ "type": "integer",
57+ "primaryKey": false,
58+ "notNull": false
59+ },
60+ "tool_call_counts": {
61+ "name": "tool_call_counts",
62+ "type": "text",
63+ "primaryKey": false,
64+ "notNull": false
65+ }
66+ },
67+ "indexes": {},
68+ "foreignKeys": {
69+ "messages_thread_id_threads_id_fk": {
70+ "name": "messages_thread_id_threads_id_fk",
71+ "tableFrom": "messages",
72+ "tableTo": "threads",
73+ "columnsFrom": [
74+ "thread_id"
75+ ],
76+ "columnsTo": [
77+ "id"
78+ ],
79+ "onDelete": "cascade",
80+ "onUpdate": "no action"
81+ }
82+ },
83+ "compositePrimaryKeys": {},
84+ "uniqueConstraints": {},
85+ "policies": {},
86+ "checkConstraints": {},
87+ "isRLSEnabled": false
88+ },
89+ "public.threads": {
90+ "name": "threads",
91+ "schema": "",
92+ "columns": {
93+ "id": {
94+ "name": "id",
95+ "type": "serial",
96+ "primaryKey": true,
97+ "notNull": true
98+ },
99+ "title": {
100+ "name": "title",
101+ "type": "text",
102+ "primaryKey": false,
103+ "notNull": false
104+ },
105+ "model": {
106+ "name": "model",
107+ "type": "text",
108+ "primaryKey": false,
109+ "notNull": true,
110+ "default": "'gpt-oss'"
111+ },
112+ "max_prompt_length": {
113+ "name": "max_prompt_length",
114+ "type": "integer",
115+ "primaryKey": false,
116+ "notNull": false
117+ },
118+ "created_at": {
119+ "name": "created_at",
120+ "type": "timestamp with time zone",
121+ "primaryKey": false,
122+ "notNull": true,
123+ "default": "now()"
124+ },
125+ "updated_at": {
126+ "name": "updated_at",
127+ "type": "timestamp with time zone",
128+ "primaryKey": false,
129+ "notNull": true,
130+ "default": "now()"
131+ }
132+ },
133+ "indexes": {},
134+ "foreignKeys": {},
135+ "compositePrimaryKeys": {},
136+ "uniqueConstraints": {},
137+ "policies": {},
138+ "checkConstraints": {},
139+ "isRLSEnabled": false
140+ }
141+ },
142+ "enums": {},
143+ "schemas": {},
144+ "sequences": {},
145+ "roles": {},
146+ "policies": {},
147+ "views": {},
148+ "_meta": {
149+ "columns": {},
150+ "schemas": {},
151+ "tables": {}
152+ }
153+}
src/lib/ollama/chat.ts+190−0
@@ -0,0 +1,190 @@
1+/* personal-assistant-thing/src/lib/chat.ts */
2+import { fetchOllamaResponse } from "@/src/lib/ollama";
3+import type { MaxPromptLength } from "@/src/lib/ollama/types";
4+import type { Message, ToolCall } from "ollama";
5+import { db } from "@/src/lib/db";
6+import { messages, threads } from "@/src/lib/db/schema";
7+import { eq } from "drizzle-orm";
8+
9+/**
10+ * Extracts tool name counts from an array of tool calls.
11+ * Returns a JSON string of the format: {"tool_name": count, ...}
12+ */
13+function extractToolCounts(toolCalls: ToolCall[]): string | null {
14+ if (toolCalls.length === 0) return null;
15+
16+ const toolCounts: Record<string, number> = {};
17+ for (const toolCall of toolCalls) {
18+ const toolName = toolCall.function.name;
19+ toolCounts[toolName] = (toolCounts[toolName] || 0) + 1;
20+ }
21+
22+ return JSON.stringify(toolCounts);
23+}
24+
25+export interface StreamAssistantResponseParams {
26+ ollamaMessages: Message[];
27+ threadId: number;
28+ threadModel: string | null;
29+ threadMaxPromptLength: MaxPromptLength;
30+ signal: AbortSignal;
31+}
32+
33+/**
34+ * Streams assistant response and handles database persistence
35+ */
36+export function streamAssistantResponse({
37+ ollamaMessages,
38+ threadId,
39+ threadModel,
40+ threadMaxPromptLength,
41+ signal,
42+}: StreamAssistantResponseParams): ReadableStream<Uint8Array> {
43+ let accumulatedContent = "";
44+ const allToolCalls: ToolCall[] = [];
45+
46+ const encoder = new TextEncoder();
47+ const stream = new ReadableStream({
48+ async start(controller) {
49+ // Track if controller is closed to avoid double-close
50+ let isControllerClosed = false;
51+ const safeClose = () => {
52+ if (!isControllerClosed) {
53+ isControllerClosed = true;
54+ try {
55+ controller.close();
56+ } catch {
57+ // Controller may already be closed, ignore
58+ }
59+ }
60+ };
61+
62+ const safeEnqueue = (data: unknown) => {
63+ if (!isControllerClosed) {
64+ try {
65+ controller.enqueue(encoder.encode(`data: ${JSON.stringify(data)}\n\n`));
66+ } catch {
67+ // Controller may be closed, ignore
68+ }
69+ }
70+ };
71+
72+ try {
73+ // Check if already aborted
74+ if (signal.aborted) {
75+ safeClose();
76+ return;
77+ }
78+
79+ const onChunk = (chunk: {
80+ content?: string;
81+ thinking?: string;
82+ toolCalls?: ToolCall[];
83+ }) => {
84+ // Check abort signal before each chunk
85+ if (signal.aborted || isControllerClosed) {
86+ return;
87+ }
88+
89+ if (chunk.content) {
90+ accumulatedContent += chunk.content;
91+ safeEnqueue({ type: "content", content: chunk.content });
92+ }
93+
94+ if (chunk.toolCalls) {
95+ allToolCalls.push(...chunk.toolCalls);
96+ safeEnqueue({ type: "toolCalls", toolCalls: chunk.toolCalls });
97+ }
98+ };
99+
100+ // Set up abort handler
101+ const abortHandler = () => {
102+ safeEnqueue({ type: "error", error: "Generation stopped" });
103+ safeClose();
104+ };
105+ signal.addEventListener("abort", abortHandler);
106+
107+ let generationTimeMs: number | undefined;
108+ let toolCalls: ToolCall[] | undefined;
109+
110+ try {
111+ const result = await fetchOllamaResponse(
112+ ollamaMessages,
113+ onChunk,
114+ threadModel,
115+ signal,
116+ threadMaxPromptLength
117+ );
118+ generationTimeMs = result.generationTimeMs;
119+ toolCalls = result.toolCalls;
120+ } catch (error) {
121+ // If aborted, don't throw - just close
122+ if (signal.aborted || (error instanceof Error && error.message === "Request aborted")) {
123+ safeClose();
124+ return;
125+ }
126+ throw error;
127+ } finally {
128+ signal.removeEventListener("abort", abortHandler);
129+ }
130+
131+ // Check if aborted before saving
132+ if (signal.aborted) {
133+ // Save partial content if any was generated
134+ if (accumulatedContent) {
135+ await db.insert(messages).values({
136+ threadId: threadId,
137+ role: "assistant",
138+ content: accumulatedContent,
139+ model: threadModel,
140+ maxPromptLength: threadMaxPromptLength === "none" ? null : threadMaxPromptLength, // null means "none" (no limit), otherwise 1024 or 4096
141+ createdAt: new Date(),
142+ generationTimeMs: null,
143+ toolCallCounts: extractToolCounts(allToolCalls),
144+ });
145+ await db.update(threads).set({ updatedAt: new Date() }).where(eq(threads.id, threadId));
146+
147+ // Send done message with metadata for aborted generation
148+ safeEnqueue({
149+ type: "done",
150+ answer: accumulatedContent,
151+ model: threadModel,
152+ maxPromptLength: threadMaxPromptLength,
153+ });
154+ }
155+ safeClose();
156+ return;
157+ }
158+
159+ // Store assistant message with model, maxPromptLength, generation time and tool calls
160+ await db.insert(messages).values({
161+ threadId: threadId,
162+ role: "assistant",
163+ content: accumulatedContent,
164+ model: threadModel,
165+ maxPromptLength: threadMaxPromptLength === "none" ? null : threadMaxPromptLength, // null means "none" (no limit), otherwise 1024 or 4096
166+ createdAt: new Date(),
167+ generationTimeMs,
168+ toolCallCounts: toolCalls ? extractToolCounts(toolCalls) : null,
169+ });
170+
171+ // Update thread's updatedAt timestamp
172+ await db.update(threads).set({ updatedAt: new Date() }).where(eq(threads.id, threadId));
173+
174+ // Send final message with metadata
175+ safeEnqueue({
176+ type: "done",
177+ answer: accumulatedContent,
178+ model: threadModel,
179+ maxPromptLength: threadMaxPromptLength,
180+ });
181+ safeClose();
182+ } catch {
183+ safeEnqueue({ type: "error", error: "Sorry, something went wrong." });
184+ safeClose();
185+ }
186+ },
187+ });
188+
189+ return stream;
190+}
src/lib/ollama/duckduckgo.ts+20−60
@@ -12,11 +12,7 @@ const duckDuckGoRateLimiter = createRateLimiter({
1212 });
1313
1414 /**
15- * Queries DuckDuckGo's Instant Answer API for information about a given query.
16- * Results are cached for 6 hours.
17- *
18- * @param query - The search query to send to DuckDuckGo.
19- * @returns A formatted string containing the instant answer information or an error message.
15+ * Queries DuckDuckGo Instant Answer API. Results cached 6h.
2016 */
2117 export async function queryDuckDuckGo(query: string): Promise<string> {
2218 // Check cache first
@@ -29,17 +25,11 @@ export async function queryDuckDuckGo(query: string): Promise<string> {
2925
3026 debug(`[DuckDuckGo] Querying: "${query}"`);
3127
32- // Check rate limit before making the request
3328 const rateLimitCheck = await duckDuckGoRateLimiter.check();
3429 if (!rateLimitCheck.allowed) {
35- const hoursRemaining = rateLimitCheck.hoursUntilReset || 0;
36- throw new Error(
37- `DuckDuckGo API rate limit exceeded. Maximum of 500 requests per 24 hours has been reached. Please try again in approximately ${hoursRemaining} hour${hoursRemaining !== 1 ? "s" : ""}.`
38- );
30+ throw new Error("Service temporarily unavailable");
3931 }
4032
41- // Increment rate limit counter before making the request
42- // This ensures all API attempts are counted, not just successful ones
4333 await duckDuckGoRateLimiter.increment();
4434 const url = "https://api.duckduckgo.com/";
4535 const params = new URLSearchParams({
@@ -52,79 +42,49 @@ export async function queryDuckDuckGo(query: string): Promise<string> {
5242 const response = await fetch(`${url}?${params.toString()}`);
5343 if (!response.ok) {
5444 debug(`[DuckDuckGo] Error: HTTP ${response.status}`);
55- throw new Error(`Failed to fetch data from DuckDuckGo (${response.status})`);
45+ throw new Error("Service error");
5646 }
5747
5848 const data = await response.json();
5949
60- // Build a comprehensive response from available fields
6150 const parts: string[] = [];
62-
63- if (data.Heading) {
64- parts.push(`Heading: ${data.Heading}`);
65- }
66-
67- if (data.AbstractText) {
68- parts.push(`Abstract: ${data.AbstractText}`);
69- }
70-
71- if (data.AbstractURL) {
72- parts.push(`Source: ${data.AbstractURL}`);
73- }
74-
75- if (data.Answer) {
76- parts.push(`Answer: ${data.Answer}`);
77- }
78-
79- if (data.Definition) {
80- parts.push(`Definition: ${data.Definition}`);
81- }
82-
83- if (data.Type) {
84- parts.push(`Type: ${data.Type}`);
85- }
86-
87- if (data.RelatedTopics && data.RelatedTopics.length > 0) {
88- const topics = data.RelatedTopics.slice(0, 3)
89- .map((topic: { Text?: string; FirstURL?: string }) => topic.Text || topic.FirstURL)
90- .filter(Boolean)
91- .join(", ");
92- if (topics) {
93- parts.push(`Related Topics: ${topics}`);
94- }
95- }
51+ if (data.Answer) parts.push(data.Answer);
52+ if (data.AbstractText) parts.push(data.AbstractText);
53+ if (data.Definition) parts.push(data.Definition);
54+ if (data.Heading && !parts.length) parts.push(data.Heading);
9655
9756 if (parts.length === 0) {
98- // DuckDuckGo Instant Answer API has limitations - it doesn't support all query types
99- // (e.g., weather forecasts, real-time data). This is expected behavior.
10057 debug(`[DuckDuckGo] No instant answer data available for this query type`);
101- const result = `No instant answer available for query: "${query}". Note: DuckDuckGo Instant Answer API has limited coverage and may not support weather forecasts, real-time data, or certain query types.`;
102- // Cache even "no answer" responses to avoid repeated API calls
58+ const result = `No answer found for: ${query}`;
10359 await setCache(cacheKey, result, CACHE_TTL_MS);
10460 return result;
10561 }
10662
107- const result = parts.join("\n\n");
108- // Cache successful results
63+ let result = parts[0];
64+ if (data.AbstractURL) result += ` (${data.AbstractURL})`;
65+ if (data.RelatedTopics?.length > 0) {
66+ const topics = data.RelatedTopics.slice(0, 2)
67+ .map((t: { Text?: string }) => t.Text)
68+ .filter(Boolean)
69+ .join(", ");
70+ if (topics) result += ` | Related: ${topics}`;
71+ }
72+
10973 await setCache(cacheKey, result, CACHE_TTL_MS);
11074 return result;
11175 }
11276
113-/**
114- * Defines the DuckDuckGo search tool for Ollama.
115- */
11677 export const duckDuckGoTool = {
11778 type: "function" as const,
11879 function: {
11980 name: "query_duckduckgo",
120- description:
121- "Query DuckDuckGo's Instant Answer API to get quick information about a topic, person, place, or concept. Use this when you need current or factual information that might not be in your training data.",
81+ description: "Get quick info from DuckDuckGo about topics, people, places, or concepts.",
12282 parameters: {
12383 type: "object",
12484 properties: {
12585 query: {
12686 type: "string",
127- description: "The search query to send to DuckDuckGo",
87+ description: "Search query",
12888 },
12989 },
13090 required: ["query"],
src/lib/ollama/websearch.ts+19−46
@@ -7,10 +7,6 @@ import { createRateLimiter } from "../ratelimit";
77
88 const CACHE_TTL_MS = 6 * 60 * 60 * 1000; // 6 hours
99
10-/**
11- * Rate limiter for web search requests.
12- * Tracks requests per 24-hour rolling window.
13- */
1410 const webSearchRateLimiter = createRateLimiter({
1511 maxRequests: 100,
1612 windowMs: 24 * 60 * 60 * 1000, // 24 hours in milliseconds
@@ -18,12 +14,7 @@ const webSearchRateLimiter = createRateLimiter({
1814 });
1915
2016 /**
21- * Performs a web search using Google Custom Search API.
22- * Enforces a rate limit of 100 requests per 24 hours.
23- * Results are cached for 6 hours.
24- *
25- * @param query - The search query to execute on the web.
26- * @returns A formatted string containing search results or an error message.
17+ * Performs web search via Google Custom Search API. Results cached 6h.
2718 */
2819 export async function queryWebSearch(query: string): Promise<string> {
2920 // Check cache first
@@ -41,22 +32,14 @@ export async function queryWebSearch(query: string): Promise<string> {
4132
4233 if (!apiKey || !cx) {
4334 debug(`[WebSearch] Missing API credentials`);
44- throw new Error(
45- "Google Custom Search API credentials not configured. Please set GOOGLE_CUSTOM_SEARCH_API_KEY and GOOGLE_CUSTOM_SEARCH_ENGINE_ID environment variables."
46- );
35+ throw new Error("Service unavailable");
4736 }
4837
49- // Check rate limit before making the request
5038 const rateLimitCheck = await webSearchRateLimiter.check();
5139 if (!rateLimitCheck.allowed) {
52- const hoursRemaining = rateLimitCheck.hoursUntilReset || 0;
53- throw new Error(
54- `Web search rate limit exceeded. Maximum of 100 requests per 24 hours has been reached. Please try again in approximately ${hoursRemaining} hour${hoursRemaining !== 1 ? "s" : ""}.`
55- );
40+ throw new Error("Service temporarily unavailable");
5641 }
5742
58- // Increment rate limit counter before making the request
59- // This ensures all API attempts are counted, not just successful ones
6043 await webSearchRateLimiter.increment();
6144
6245 const customsearch = google.customsearch("v1");
@@ -68,49 +51,39 @@ export async function queryWebSearch(query: string): Promise<string> {
6851
6952 if (!res.data.items || res.data.items.length === 0) {
7053 debug(`[WebSearch] No results found for query: "${query}"`);
71- const result = `No search results found for query: "${query}"`;
72- // Cache even "no results" responses to avoid repeated API calls
54+ const result = `No results for: ${query}`;
7355 await setCache(cacheKey, result, CACHE_TTL_MS);
7456 return result;
7557 }
7658
7759 debug(`[WebSearch] Found ${res.data.items.length} results`);
7860
79- const parts: string[] = [];
80- parts.push(`Web search results for "${query}":\n`);
81-
82- res.data.items.forEach((item, index) => {
83- parts.push(`${index + 1}. ${item.title || "Untitled"}`);
84- if (item.link) {
85- parts.push(` URL: ${item.link}`);
86- }
87- if (item.snippet) {
88- parts.push(` ${item.snippet}`);
89- }
90- parts.push(""); // Empty line between results
91- });
92-
93- const result = parts.join("\n").trim();
94- // Cache successful results
95- await setCache(cacheKey, result, CACHE_TTL_MS);
96- return result;
61+ const results = res.data.items
62+ .slice(0, 5)
63+ .map((item) => {
64+ const parts: string[] = [];
65+ if (item.title) parts.push(item.title);
66+ if (item.snippet) parts.push(item.snippet);
67+ if (item.link) parts.push(`(${item.link})`);
68+ return parts.join(" | ");
69+ })
70+ .join("\n");
71+
72+ await setCache(cacheKey, results, CACHE_TTL_MS);
73+ return results;
9774 }
9875
99-/**
100- * Defines the web search tool for Ollama.
101- */
10276 export const webSearchTool = {
10377 type: "function" as const,
10478 function: {
10579 name: "query_web_search",
106- description:
107- "Perform a web search using Google Custom Search API to find current information, news, articles, or any content on the internet. Use this when you need to find recent information, verify facts, or search for content that may not be in your training data.",
80+ description: "Search the web for current info, news, articles, or any content.",
10881 parameters: {
10982 type: "object",
11083 properties: {
11184 query: {
11285 type: "string",
113- description: "The search query to execute on the web",
86+ description: "Search query",
11487 },
11588 },
11689 required: ["query"],