53129
Pa
PAT / files
PAT / src / lib / tools / fetchpage.ts
2.0 KB
1import { getCache, setCache } from "@/src/lib/cache"; 2import { debug } from "@/src/lib/debug"; 3import { parse } from "node-html-better-parser"; 4 5const CACHE_TTL_MS = 6 * 60 * 60 * 1000; // 6 hours 6 7type FetchPageResult = { 8 title: string; 9 content: string; 10 links: string[]; 11}; 12 13const extractText = (el: unknown): string => { 14 if (!el || typeof el !== "object") return ""; 15 const elem = el as { text?: string; childNodes?: unknown[] }; 16 let text = elem.text || ""; 17 if (elem.childNodes) { 18 for (const child of elem.childNodes) { 19 text += extractText(child); 20 } 21 } 22 return text; 23}; 24 25export async function fetchPage(url: string): Promise<FetchPageResult> { 26 const cached = await getCache<FetchPageResult>(`fetchpage:${url}`); 27 if (cached) { 28 return cached; 29 } 30 31 const result: FetchPageResult = { 32 title: "", 33 content: "", 34 links: [], 35 }; 36 37 const response = await fetch(url); 38 const html = await response.text(); 39 const root = parse(html); 40 41 const title = root.querySelector("title")?.text; 42 const content = extractText(root.querySelector("body")); 43 const links = root.querySelectorAll("a").map((a) => a.getAttribute("href")); 44 45 result.title = title?.trim() || ""; 46 result.content = content?.trim() || ""; 47 result.links = links.filter((link): link is string => link !== null); 48 49 debug(`fetched page ${url}`, result); 50 51 await setCache<FetchPageResult>(`fetchpage:${url}`, result, CACHE_TTL_MS); 52 53 return result; 54} 55 56export const fetchPageTool = { 57 type: "function" as const, 58 function: { 59 name: "fetch_page", 60 description: 61 "Fetch a web page and return the title, content, and links in JSON format. Use this tool when asked to fetch a web page, retrieve content from a URL, or read information from a specific webpage.", 62 parameters: { 63 type: "object", 64 properties: { 65 url: { 66 type: "string", 67 description: "The URL of the web page to fetch (must be HTTP or HTTPS).", 68 }, 69 }, 70 required: ["url"], 71 }, 72 }, 73};