xonecas.io
53129
Pa

PAT / files

PAT / src / lib / tools / fetchpage.ts

2.0 KB
1import { getCache, setCache } from "@/src/lib/cache";
2import { debug } from "@/src/lib/debug";
3import { parse } from "node-html-better-parser";
4
5const CACHE_TTL_MS = 6 * 60 * 60 * 1000; // 6 hours
6
7type FetchPageResult = {
8  title: string;
9  content: string;
10  links: string[];
11};
12
13const extractText = (el: unknown): string => {
14  if (!el || typeof el !== "object") return "";
15  const elem = el as { text?: string; childNodes?: unknown[] };
16  let text = elem.text || "";
17  if (elem.childNodes) {
18    for (const child of elem.childNodes) {
19      text += extractText(child);
20    }
21  }
22  return text;
23};
24
25export async function fetchPage(url: string): Promise<FetchPageResult> {
26  const cached = await getCache<FetchPageResult>(`fetchpage:${url}`);
27  if (cached) {
28    return cached;
29  }
30
31  const result: FetchPageResult = {
32    title: "",
33    content: "",
34    links: [],
35  };
36
37  const response = await fetch(url);
38  const html = await response.text();
39  const root = parse(html);
40
41  const title = root.querySelector("title")?.text;
42  const content = extractText(root.querySelector("body"));
43  const links = root.querySelectorAll("a").map((a) => a.getAttribute("href"));
44
45  result.title = title?.trim() || "";
46  result.content = content?.trim() || "";
47  result.links = links.filter((link): link is string => link !== null);
48
49  debug(`fetched page ${url}`, result);
50
51  await setCache<FetchPageResult>(`fetchpage:${url}`, result, CACHE_TTL_MS);
52
53  return result;
54}
55
56export const fetchPageTool = {
57  type: "function" as const,
58  function: {
59    name: "fetch_page",
60    description:
61      "Fetch a web page and return the title, content, and links in JSON format. Use this tool when asked to fetch a web page, retrieve content from a URL, or read information from a specific webpage.",
62    parameters: {
63      type: "object",
64      properties: {
65        url: {
66          type: "string",
67          description: "The URL of the web page to fetch (must be HTTP or HTTPS).",
68        },
69      },
70      required: ["url"],
71    },
72  },
73};