@@ -1,147 +1,56 @@
11 import { getCache, setCache } from "@/src/lib/cache";
2-import { createRateLimiter } from "@/src/lib/ratelimit";
2+import { debug } from "@/src/lib/debug";
3+import { parse } from "node-html-better-parser";
34
45 const CACHE_TTL_MS = 6 * 60 * 60 * 1000; // 6 hours
56
6-const fetchPageRateLimiter = createRateLimiter({
7- maxRequests: 100,
8- windowMs: 24 * 60 * 60 * 1000, // 24 hours in milliseconds
9- identifier: "fetchpage",
10-});
11-
12-/**
13- * Sanitizes HTML content by extracting text and removing scripts, styles, and other non-content elements.
14- * Returns a clean text string with normalized whitespace.
15- */
16-function sanitizeHtml(html: string): string {
17- // Remove script and style tags and their content
18- let sanitized = html.replace(/<script[^>]*>[\s\S]*?<\/script>/gi, "");
19- sanitized = sanitized.replace(/<style[^>]*>[\s\S]*?<\/style>/gi, "");
20- sanitized = sanitized.replace(/<noscript[^>]*>[\s\S]*?<\/noscript>/gi, "");
21-
22- // Remove HTML comments
23- sanitized = sanitized.replace(/<!--[\s\S]*?-->/g, "");
24-
25- // Extract text from common content tags
26- const textMatches = sanitized.match(
27- /<(?:p|div|h[1-6]|article|section|main|header|footer|li|td|th|span|a|strong|em|b|i|blockquote|pre|code)[^>]*>([\s\S]*?)<\/[^>]+>/gi
28- );
29-
30- if (!textMatches || textMatches.length === 0) {
31- // Fallback: extract all text between tags
32- sanitized = sanitized.replace(/<[^>]+>/g, " ");
33- } else {
34- // Extract text content from matched tags
35- sanitized = textMatches
36- .map((match) => {
37- // Remove nested tags and extract text
38- return match.replace(/<[^>]+>/g, " ").trim();
39- })
40- .join("\n");
41- }
42-
43- // Remove remaining HTML tags
44- sanitized = sanitized.replace(/<[^>]+>/g, " ");
45-
46- // Decode common HTML entities
47- sanitized = sanitized
48- .replace(/ /g, " ")
49- .replace(/&/g, "&")
50- .replace(/</g, "<")
51- .replace(/>/g, ">")
52- .replace(/"/g, '"')
53- .replace(/'/g, "'")
54- .replace(/'/g, "'");
55-
56- // Normalize whitespace
57- sanitized = sanitized
58- .replace(/\s+/g, " ")
59- .replace(/\n\s*\n/g, "\n")
60- .trim();
61-
62- return sanitized;
63-}
7+type FetchPageResult = {
8+ title: string;
9+ content: string;
10+ links: string[];
11+};
6412
65-/**
66- * Fetches a web page from the given URL and returns sanitized text content.
67- * Results are cached for 6 hours.
68- *
69- * @param url - The URL of the web page to fetch.
70- * @returns A sanitized string containing the text content of the page.
71- */
72-export async function fetchPage(url: string): Promise<string> {
73- // Validate URL
74- let parsedUrl: URL;
75- try {
76- parsedUrl = new URL(url);
77- // Only allow http and https protocols
78- if (!["http:", "https:"].includes(parsedUrl.protocol)) {
79- throw new Error("Only HTTP and HTTPS URLs are allowed");
13+const extractText = (el: unknown): string => {
14+ if (!el || typeof el !== "object") return "";
15+ const elem = el as { text?: string; childNodes?: unknown[] };
16+ let text = elem.text || "";
17+ if (elem.childNodes) {
18+ for (const child of elem.childNodes) {
19+ text += extractText(child);
8020 }
81- } catch {
82- throw new Error(`Invalid URL: ${url}`);
8321 }
22+ return text;
23+};
8424
85- // Check cache first
86- const cacheKey = `fetchpage:${url}`;
87- const cached = await getCache<string>(cacheKey);
88- if (cached !== null) {
25+export async function fetchPage(url: string): Promise<FetchPageResult> {
26+ const cached = await getCache<FetchPageResult>(`fetchpage:${url}`);
27+ if (cached) {
8928 return cached;
9029 }
9130
92- // Check rate limit
93- const rateLimitCheck = await fetchPageRateLimiter.check();
94- if (!rateLimitCheck.allowed) {
95- throw new Error("Service temporarily unavailable");
96- }
97-
98- await fetchPageRateLimiter.increment();
99-
100- try {
101- // Fetch the page with a timeout and user agent
102- const controller = new AbortController();
103- const timeoutId = setTimeout(() => controller.abort(), 10000); // 10 second timeout
104-
105- const response = await fetch(url, {
106- signal: controller.signal,
107- headers: {
108- "User-Agent": "Mozilla/5.0 (compatible; PersonalAssistant/1.0)",
109- },
110- });
31+ const result: FetchPageResult = {
32+ title: "",
33+ content: "",
34+ links: [],
35+ };
11136
112- clearTimeout(timeoutId);
37+ const response = await fetch(url);
38+ const html = await response.text();
39+ const root = parse(html);
11340
114- if (!response.ok) {
115- throw new Error(`Failed to fetch page: ${response.status} ${response.statusText}`);
116- }
41+ const title = root.querySelector("title")?.text;
42+ const content = extractText(root.querySelector("body"));
43+ const links = root.querySelectorAll("a").map((a) => a.getAttribute("href"));
11744
118- const html = await response.text();
119- const sanitized = sanitizeHtml(html);
45+ result.title = title?.trim() || "";
46+ result.content = content?.trim() || "";
47+ result.links = links.filter((link): link is string => link !== null);
12048
121- if (!sanitized || sanitized.trim().length === 0) {
122- const result = `No text content found at: ${url}`;
123- await setCache(cacheKey, result, CACHE_TTL_MS);
124- return result;
125- }
49+ debug(`fetched page ${url}`, result);
12650
127- // Limit result length to prevent excessive token usage
128- const maxLength = 10000;
129- const result =
130- sanitized.length > maxLength
131- ? `${sanitized.substring(0, maxLength)}... [content truncated]`
132- : sanitized;
51+ await setCache<FetchPageResult>(`fetchpage:${url}`, result, CACHE_TTL_MS);
13352
134- await setCache(cacheKey, result, CACHE_TTL_MS);
135- return result;
136- } catch (error) {
137- if (error instanceof Error) {
138- if (error.name === "AbortError") {
139- throw new Error("Request timeout: The page took too long to load");
140- }
141- throw error;
142- }
143- throw new Error("Unknown error occurred while fetching the page");
144- }
53+ return result;
14554 }
14655
14756 export const fetchPageTool = {
@@ -149,7 +58,7 @@ export const fetchPageTool = {
14958 function: {
15059 name: "fetch_page",
15160 description:
152- "Fetches a web page from a given URL and returns sanitized text content. Use this tool when asked to fetch a web page, retrieve content from a URL, or read information from a specific webpage.",
61+ "Fetch a web page and return the title, content, and links in JSON format. Use this tool when asked to fetch a web page, retrieve content from a URL, or read information from a specific webpage.",
15362 parameters: {
15463 type: "object",
15564 properties: {