xonecas.io
5414
Sa

sat / files

sat / src / lib / fetchURLContents.ts

2.3 KB
1import * as cheerio from "cheerio";
2
3export async function fetchURLContents(url: string): Promise<string> {
4  const response = await fetch(url, {
5    headers: {
6      "User-Agent": "Mozilla/5.0 (compatible; LLMFetcher/1.0)",
7      Accept: "text/html",
8    },
9    signal: AbortSignal.timeout(10000),
10  });
11
12  if (!response.ok) return "Failed to fetch URL";
13
14  const html = await response.text();
15  const $ = cheerio.load(html);
16
17  $(
18    // Expanded list to include more non-content/boilerplate tags
19    "head, img, script, style, link, noscript, iframe, svg, nav, footer, header, form, input, button, select, option, label, canvas, figure, figcaption, object, embed, video, audio, source, track, picture, map, area, meta, base, col, colgroup, frame, frameset, param, dialog, template, menu, menuitem, output, progress",
20  ).remove();
21
22  // Traverse body tag's children and extract text content
23  let cleanText = "";
24  const body = $("body");
25
26  function traverse(element: ReturnType<typeof $>) {
27    element.contents().each((_, node) => {
28      if (node.type === "text") {
29        const text = $(node).text().trim();
30        if (text) {
31          cleanText += `${text} `;
32        }
33      } else if (node.type === "tag") {
34        const $node = $(node);
35        const tagName = node.name;
36
37        // Convert links to markdown format [Text](URL)
38        if (tagName === "a") {
39          const linkText = $node.text().trim();
40          const linkHref = $node.attr("href");
41          if (linkText && linkHref) {
42            try {
43              const absoluteUrl = new URL(linkHref, url).href;
44              cleanText += `[${linkText}](${absoluteUrl}) `;
45            } catch {
46              cleanText += `${linkText} `;
47            }
48          }
49          return; // Don't traverse into link children
50        }
51
52        // Add line breaks for block-level elements
53        if (
54          ["p", "div", "h1", "h2", "h3", "h4", "h5", "h6", "li", "br"].includes(
55            tagName,
56          )
57        ) {
58          traverse($node);
59          cleanText += "\n\n";
60        } else {
61          traverse($node);
62        }
63      }
64    });
65  }
66
67  traverse(body);
68
69  // 5. Final Cleanup: Normalize whitespace and remove excessive newlines
70  return cleanText
71    .replace(/[ \t]+/g, " ") // Collapse multiple spaces/tabs
72    .replace(/\n\s*\n/g, "\n\n") // Collapse multiple newlines
73    .trim();
74}