@@ -0,0 +1,171 @@
1+/* personal-assistant-thing/src/lib/ollama/fetchpage.ts */
2+
3+import { getCache, setCache } from "../cache";
4+import { createRateLimiter } from "../ratelimit";
5+
6+const CACHE_TTL_MS = 6 * 60 * 60 * 1000; // 6 hours
7+
8+const fetchPageRateLimiter = createRateLimiter({
9+ maxRequests: 100,
10+ windowMs: 24 * 60 * 60 * 1000, // 24 hours in milliseconds
11+ identifier: "fetchpage",
12+});
13+
14+/**
15+ * Sanitizes HTML content by extracting text and removing scripts, styles, and other non-content elements.
16+ * Returns a clean text string with normalized whitespace.
17+ */
18+function sanitizeHtml(html: string): string {
19+ // Remove script and style tags and their content
20+ let sanitized = html.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "");
21+ sanitized = sanitized.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "");
22+ sanitized = sanitized.replace(/<noscript[^>]*>[\s\S]*?<\/noscript>/gi, "");
23+
24+ // Remove HTML comments
25+ sanitized = sanitized.replace(/<!--[\s\S]*?-->/g, "");
26+
27+ // Extract text from common content tags
28+ const textMatches = sanitized.match(
29+ /<(?:p|div|h[1-6]|article|section|main|header|footer|li|td|th|span|a|strong|em|b|i|blockquote|pre|code)[^>]*>([\s\S]*?)<\/[^>]+>/gi
30+ );
31+
32+ if (!textMatches || textMatches.length === 0) {
33+ // Fallback: extract all text between tags
34+ sanitized = sanitized.replace(/<[^>]+>/g, " ");
35+ } else {
36+ // Extract text content from matched tags
37+ sanitized = textMatches
38+ .map((match) => {
39+ // Remove nested tags and extract text
40+ return match.replace(/<[^>]+>/g, " ").trim();
41+ })
42+ .join("\n");
43+ }
44+
45+ // Remove remaining HTML tags
46+ sanitized = sanitized.replace(/<[^>]+>/g, " ");
47+
48+ // Decode common HTML entities
49+ sanitized = sanitized
50+ .replace(/ /g, " ")
51+ .replace(/&/g, "&")
52+ .replace(/</g, "<")
53+ .replace(/>/g, ">")
54+ .replace(/"/g, '"')
55+ .replace(/'/g, "'")
56+ .replace(/'/g, "'");
57+
58+ // Normalize whitespace
59+ sanitized = sanitized
60+ .replace(/\s+/g, " ")
61+ .replace(/\n\s*\n/g, "\n")
62+ .trim();
63+
64+ return sanitized;
65+}
66+
67+/**
68+ * Fetches a web page from the given URL and returns sanitized text content.
69+ * Results are cached for 6 hours.
70+ *
71+ * @param url - The URL of the web page to fetch.
72+ * @returns A sanitized string containing the text content of the page.
73+ */
74+export async function fetchPage(url: string): Promise<string> {
75+ // Validate URL
76+ let parsedUrl: URL;
77+ try {
78+ parsedUrl = new URL(url);
79+ // Only allow http and https protocols
80+ if (!["http:", "https:"].includes(parsedUrl.protocol)) {
81+ throw new Error("Only HTTP and HTTPS URLs are allowed");
82+ }
83+ } catch {
84+ throw new Error(`Invalid URL: ${url}`);
85+ }
86+
87+ // Check cache first
88+ const cacheKey = `fetchpage:${url}`;
89+ const cached = await getCache<string>(cacheKey);
90+ if (cached !== null) {
91+ return cached;
92+ }
93+
94+ // Check rate limit
95+ const rateLimitCheck = await fetchPageRateLimiter.check();
96+ if (!rateLimitCheck.allowed) {
97+ throw new Error("Service temporarily unavailable");
98+ }
99+
100+ await fetchPageRateLimiter.increment();
101+
102+ try {
103+ // Fetch the page with a timeout and user agent
104+ const controller = new AbortController();
105+ const timeoutId = setTimeout(() => controller.abort(), 10000); // 10 second timeout
106+
107+ const response = await fetch(url, {
108+ signal: controller.signal,
109+ headers: {
110+ "User-Agent": "Mozilla/5.0 (compatible; PersonalAssistant/1.0)",
111+ },
112+ });
113+
114+ clearTimeout(timeoutId);
115+
116+ if (!response.ok) {
117+ throw new Error(`Failed to fetch page: ${response.status} ${response.statusText}`);
118+ }
119+
120+ const contentType = response.headers.get("content-type") || "";
121+ if (!contentType.includes("text/html")) {
122+ throw new Error(`Unsupported content type: ${contentType}`);
123+ }
124+
125+ const html = await response.text();
126+ const sanitized = sanitizeHtml(html);
127+
128+ if (!sanitized || sanitized.trim().length === 0) {
129+ const result = `No text content found at: ${url}`;
130+ await setCache(cacheKey, result, CACHE_TTL_MS);
131+ return result;
132+ }
133+
134+ // Limit result length to prevent excessive token usage
135+ const maxLength = 10000;
136+ const result =
137+ sanitized.length > maxLength
138+ ? `${sanitized.substring(0, maxLength)}... [content truncated]`
139+ : sanitized;
140+
141+ await setCache(cacheKey, result, CACHE_TTL_MS);
142+ return result;
143+ } catch (error) {
144+ if (error instanceof Error) {
145+ if (error.name === "AbortError") {
146+ throw new Error("Request timeout: The page took too long to load");
147+ }
148+ throw error;
149+ }
150+ throw new Error("Unknown error occurred while fetching the page");
151+ }
152+}
153+
154+export const fetchPageTool = {
155+ type: "function" as const,
156+ function: {
157+ name: "fetch_page",
158+ description:
159+ "Fetches a web page from a given URL and returns sanitized text content. Use this tool when asked to fetch a web page, retrieve content from a URL, or read information from a specific webpage.",
160+ parameters: {
161+ type: "object",
162+ properties: {
163+ url: {
164+ type: "string",
165+ description: "The URL of the web page to fetch (must be HTTP or HTTPS).",
166+ },
167+ },
168+ required: ["url"],
169+ },
170+ },
171+};