xonecas.io
53129
Pa

PAT / commit 612510c

feat: fetch page tool

xonecas2025-12-18 612510c33ed0ab52780c6cf029dec0a3ff958360 parent ba296a1

11 files changed

app/api/models/route.ts+1−1
@@ -9,7 +9,7 @@ export async function GET() {
99 model: model.name, // Use name as the model identifier
1010 }));
1111 return NextResponse.json({ models });
12- } catch (err) {
12+ } catch {
1313 return NextResponse.json({ error: "Failed to fetch models" }, { status: 500 });
1414 }
1515 }
app/api/settings/route.ts+2−2
@@ -24,7 +24,7 @@ export async function GET() {
2424 };
2525
2626 return NextResponse.json({ settings: defaultSettings });
27- } catch (err) {
27+ } catch {
2828 return NextResponse.json({ error: "Failed to get settings" }, { status: 500 });
2929 }
3030 }
@@ -66,7 +66,7 @@ export async function POST(request: Request) {
6666 await setCache(SETTINGS_CACHE_KEY, settings);
6767
6868 return NextResponse.json({ settings: settings });
69- } catch (err) {
69+ } catch {
7070 return NextResponse.json({ error: "Failed to set settings" }, { status: 500 });
7171 }
7272 }
app/api/threads/[id]/messages/[messageId]/route.ts+1−1
@@ -30,7 +30,7 @@ export async function DELETE(
3030 }
3131
3232 return NextResponse.json({ success: true });
33- } catch (err) {
33+ } catch {
3434 return NextResponse.json({ error: "Failed to delete message" }, { status: 500 });
3535 }
3636 }
app/api/threads/[id]/messages/route.ts+1−1
@@ -18,7 +18,7 @@ export async function GET(request: Request, { params }: { params: Promise<{ id:
1818 .orderBy(asc(messages.createdAt));
1919
2020 return NextResponse.json({ messages: messagesList });
21- } catch (err) {
21+ } catch {
2222 return NextResponse.json({ error: "Failed to get messages" }, { status: 500 });
2323 }
2424 }
app/api/threads/[id]/route.ts+2−2
@@ -38,7 +38,7 @@ export async function PATCH(request: Request, { params }: { params: Promise<{ id
3838 }
3939
4040 return NextResponse.json({ thread: updatedThread[0] });
41- } catch (err) {
41+ } catch {
4242 return NextResponse.json({ error: "Failed to update thread" }, { status: 500 });
4343 }
4444 }
@@ -59,7 +59,7 @@ export async function DELETE(request: Request, { params }: { params: Promise<{ i
5959 }
6060
6161 return NextResponse.json({ success: true });
62- } catch (err) {
62+ } catch {
6363 return NextResponse.json({ error: "Failed to delete thread" }, { status: 500 });
6464 }
6565 }
app/api/threads/route.ts+2−2
@@ -33,7 +33,7 @@ export async function POST(request: Request) {
3333 });
3434
3535 return NextResponse.json({ thread: newThread[0] });
36- } catch (err) {
36+ } catch {
3737 return NextResponse.json({ error: "Failed to create thread" }, { status: 500 });
3838 }
3939 }
@@ -56,7 +56,7 @@ export async function GET(request: Request) {
5656 const hasMore = offset + limit < totalCount;
5757
5858 return NextResponse.json({ threads: threadsList, hasMore, totalCount });
59- } catch (err) {
59+ } catch {
6060 return NextResponse.json({ error: "Failed to get threads" }, { status: 500 });
6161 }
6262 }
src/lib/cache/index.ts+2−2
@@ -23,7 +23,7 @@ export async function setCache<T>(key: string, value: T, ttlMs?: number): Promis
2323 } else {
2424 await valkey.set(key, serialized);
2525 }
26- } catch (error: unknown) {
26+ } catch {
2727 // Don't throw - cache operations should continue even if persistence fails
2828 }
2929 }
@@ -43,7 +43,7 @@ export async function getCache<T>(key: string): Promise<T | null> {
4343 }
4444
4545 return JSON.parse(serialized) as T;
46- } catch (error: unknown) {
46+ } catch {
4747 return null;
4848 }
4949 }
src/lib/ollama/fetchpage.ts+171−0
@@ -0,0 +1,171 @@
1+/* personal-assistant-thing/src/lib/ollama/fetchpage.ts */
2+
3+import { getCache, setCache } from "../cache";
4+import { createRateLimiter } from "../ratelimit";
5+
6+const CACHE_TTL_MS = 6 * 60 * 60 * 1000; // 6 hours
7+
8+const fetchPageRateLimiter = createRateLimiter({
9+ maxRequests: 100,
10+ windowMs: 24 * 60 * 60 * 1000, // 24 hours in milliseconds
11+ identifier: "fetchpage",
12+});
13+
14+/**
15+ * Sanitizes HTML content by extracting text and removing scripts, styles, and other non-content elements.
16+ * Returns a clean text string with normalized whitespace.
17+ */
18+function sanitizeHtml(html: string): string {
19+ // Remove script and style tags and their content
20+ let sanitized = html.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "");
21+ sanitized = sanitized.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "");
22+ sanitized = sanitized.replace(/<noscript[^>]*>[\s\S]*?<\/noscript>/gi, "");
23+
24+ // Remove HTML comments
25+ sanitized = sanitized.replace(/<!--[\s\S]*?-->/g, "");
26+
27+ // Extract text from common content tags
28+ const textMatches = sanitized.match(
29+ /<(?:p|div|h[1-6]|article|section|main|header|footer|li|td|th|span|a|strong|em|b|i|blockquote|pre|code)[^>]*>([\s\S]*?)<\/[^>]+>/gi
30+ );
31+
32+ if (!textMatches || textMatches.length === 0) {
33+ // Fallback: extract all text between tags
34+ sanitized = sanitized.replace(/<[^>]+>/g, " ");
35+ } else {
36+ // Extract text content from matched tags
37+ sanitized = textMatches
38+ .map((match) => {
39+ // Remove nested tags and extract text
40+ return match.replace(/<[^>]+>/g, " ").trim();
41+ })
42+ .join("\n");
43+ }
44+
45+ // Remove remaining HTML tags
46+ sanitized = sanitized.replace(/<[^>]+>/g, " ");
47+
48+ // Decode common HTML entities
49+ sanitized = sanitized
50+ .replace(/&nbsp;/g, " ")
51+ .replace(/&amp;/g, "&")
52+ .replace(/&lt;/g, "<")
53+ .replace(/&gt;/g, ">")
54+ .replace(/&quot;/g, '"')
55+ .replace(/&#39;/g, "'")
56+ .replace(/&apos;/g, "'");
57+
58+ // Normalize whitespace
59+ sanitized = sanitized
60+ .replace(/\s+/g, " ")
61+ .replace(/\n\s*\n/g, "\n")
62+ .trim();
63+
64+ return sanitized;
65+}
66+
67+/**
68+ * Fetches a web page from the given URL and returns sanitized text content.
69+ * Results are cached for 6 hours.
70+ *
71+ * @param url - The URL of the web page to fetch.
72+ * @returns A sanitized string containing the text content of the page.
73+ */
74+export async function fetchPage(url: string): Promise<string> {
75+ // Validate URL
76+ let parsedUrl: URL;
77+ try {
78+ parsedUrl = new URL(url);
79+ // Only allow http and https protocols
80+ if (!["http:", "https:"].includes(parsedUrl.protocol)) {
81+ throw new Error("Only HTTP and HTTPS URLs are allowed");
82+ }
83+ } catch {
84+ throw new Error(`Invalid URL: ${url}`);
85+ }
86+
87+ // Check cache first
88+ const cacheKey = `fetchpage:${url}`;
89+ const cached = await getCache<string>(cacheKey);
90+ if (cached !== null) {
91+ return cached;
92+ }
93+
94+ // Check rate limit
95+ const rateLimitCheck = await fetchPageRateLimiter.check();
96+ if (!rateLimitCheck.allowed) {
97+ throw new Error("Service temporarily unavailable");
98+ }
99+
100+ await fetchPageRateLimiter.increment();
101+
102+ try {
103+ // Fetch the page with a timeout and user agent
104+ const controller = new AbortController();
105+ const timeoutId = setTimeout(() => controller.abort(), 10000); // 10 second timeout
106+
107+ const response = await fetch(url, {
108+ signal: controller.signal,
109+ headers: {
110+ "User-Agent": "Mozilla/5.0 (compatible; PersonalAssistant/1.0)",
111+ },
112+ });
113+
114+ clearTimeout(timeoutId);
115+
116+ if (!response.ok) {
117+ throw new Error(`Failed to fetch page: ${response.status} ${response.statusText}`);
118+ }
119+
120+ const contentType = response.headers.get("content-type") || "";
121+ if (!contentType.includes("text/html")) {
122+ throw new Error(`Unsupported content type: ${contentType}`);
123+ }
124+
125+ const html = await response.text();
126+ const sanitized = sanitizeHtml(html);
127+
128+ if (!sanitized || sanitized.trim().length === 0) {
129+ const result = `No text content found at: ${url}`;
130+ await setCache(cacheKey, result, CACHE_TTL_MS);
131+ return result;
132+ }
133+
134+ // Limit result length to prevent excessive token usage
135+ const maxLength = 10000;
136+ const result =
137+ sanitized.length > maxLength
138+ ? `${sanitized.substring(0, maxLength)}... [content truncated]`
139+ : sanitized;
140+
141+ await setCache(cacheKey, result, CACHE_TTL_MS);
142+ return result;
143+ } catch (error) {
144+ if (error instanceof Error) {
145+ if (error.name === "AbortError") {
146+ throw new Error("Request timeout: The page took too long to load");
147+ }
148+ throw error;
149+ }
150+ throw new Error("Unknown error occurred while fetching the page");
151+ }
152+}
153+
154+export const fetchPageTool = {
155+ type: "function" as const,
156+ function: {
157+ name: "fetch_page",
158+ description:
159+ "Fetches a web page from a given URL and returns sanitized text content. Use this tool when asked to fetch a web page, retrieve content from a URL, or read information from a specific webpage.",
160+ parameters: {
161+ type: "object",
162+ properties: {
163+ url: {
164+ type: "string",
165+ description: "The URL of the web page to fetch (must be HTTP or HTTPS).",
166+ },
167+ },
168+ required: ["url"],
169+ },
170+ },
171+};
src/lib/ollama/index.ts+10−6
@@ -4,6 +4,7 @@
44 import ollama, { type Message, type ToolCall } from "ollama";
55 import { debug } from "@/src/lib/debug";
66 import { duckDuckGoTool, queryDuckDuckGo } from "@/src/lib/ollama/duckduckgo";
7+import { fetchPage, fetchPageTool } from "@/src/lib/ollama/fetchpage";
78 import { queryWeather, weatherTool } from "@/src/lib/ollama/weather";
89 import { queryWebSearch, webSearchTool } from "@/src/lib/ollama/websearch";
910 import type { OllamaChunk, OllamaResponse, MaxPromptLength } from "@/src/lib/ollama/types";
@@ -16,8 +17,9 @@ import type { OllamaChunk, OllamaResponse, MaxPromptLength } from "@/src/lib/oll
1617 */
1718 async function executeToolCall(toolCall: ToolCall): Promise<Message> {
1819 const { name, arguments: args } = toolCall.function;
19- const { query, timezone, forecastDays } = args as {
20- query: string;
20+ const { query, url, timezone, forecastDays } = args as {
21+ query?: string;
22+ url?: string;
2123 timezone?: string;
2224 forecastDays?: number;
2325 };
@@ -26,14 +28,17 @@ async function executeToolCall(toolCall: ToolCall): Promise<Message> {
2628 query_web_search: queryWebSearch,
2729 query_weather: queryWeather,
2830 query_duckduckgo: queryDuckDuckGo,
31+ fetch_page: fetchPage,
2932 };
3033
3134 let result: string;
3235 try {
3336 if (name === "query_weather") {
34- result = await queryWeather(query, timezone, forecastDays);
37+ result = await queryWeather(query!, timezone, forecastDays);
38+ } else if (name === "fetch_page") {
39+ result = await fetchPage(url!);
3540 } else {
36- result = await tools[name](query);
41+ result = await tools[name](query!);
3742 }
3843 debug(`[Tool] ${name} completed (${result.length} chars)`);
3944 } catch (error) {
@@ -70,7 +75,7 @@ export async function fetchOllamaResponse(
7075 signal?: AbortSignal,
7176 maxPromptLength?: MaxPromptLength
7277 ): Promise<OllamaResponse> {
73- const tools = [duckDuckGoTool, weatherTool, webSearchTool];
78+ const tools = [duckDuckGoTool, weatherTool, webSearchTool, fetchPageTool];
7479 let totalDuration = 0;
7580 const currentMessages: Message[] = messages;
7681 const allToolCalls: ToolCall[] = [];
@@ -214,6 +219,5 @@ export async function fetchOllamaResponse(
214219 });
215220 }
216221 }
217-
218222 }
219223 }
src/lib/ollama/weather.ts+2−2
@@ -151,8 +151,8 @@ export const weatherTool = {
151151 type: "function" as const,
152152 function: {
153153 name: "query_weather",
154- description:
155- "Query Open-Meteo's weather API to get current weather conditions and optionally forecast for a location. Returns temperature and precipitation data. By default returns only current weather. Use forecastDays parameter to include forecast. Use this when users ask about weather, temperature, forecast, or current conditions for any location worldwide.",
154+ description:
155+ "Query Open-Meteo's weather API to get current weather conditions and optionally forecast for a location. Returns temperature and precipitation data. By default returns only current weather. Use forecastDays parameter to include forecast. Use this when users ask about weather, temperature, forecast, or current conditions for any location worldwide.",
156156 parameters: {
157157 type: "object",
158158 properties: {
tsconfig.tsbuildinfo+1−1
@@ -1 +1 @@
1-{"fileNames":["./node_modules/typescript/lib/lib.es5.d.ts","./node_modules/typescript/lib/lib.es2015.d.ts","./node_modules/typescript/lib/lib.es2016.d.ts","./node_modules/typescript/lib/lib.es2017.d.ts","./node_modules/typescript/lib/lib.es2018.d.ts","./node_modules/typescript/lib/lib.es2019.d.ts","./node_modules/typescript/lib/lib.es2020.d.ts","./node_modules/typescript/lib/lib.es2021.d.ts","./node_modules/typescript/lib/lib.es2022.d.ts","./node_modules/typescript/lib/lib.es2023.d.ts","./node_m …
1+{"fileNames":["./node_modules/typescript/lib/lib.es5.d.ts","./node_modules/typescript/lib/lib.es2015.d.ts","./node_modules/typescript/lib/lib.es2016.d.ts","./node_modules/typescript/lib/lib.es2017.d.ts","./node_modules/typescript/lib/lib.es2018.d.ts","./node_modules/typescript/lib/lib.es2019.d.ts","./node_modules/typescript/lib/lib.es2020.d.ts","./node_modules/typescript/lib/lib.es2021.d.ts","./node_modules/typescript/lib/lib.es2022.d.ts","./node_modules/typescript/lib/lib.es2023.d.ts","./node_m …