5414
Sa
sat / files
sat / src / lib / fetchURLContents.ts
2.3 KB
1import * as cheerio from "cheerio"; 2 3export async function fetchURLContents(url: string): Promise<string> { 4 const response = await fetch(url, { 5 headers: { 6 "User-Agent": "Mozilla/5.0 (compatible; LLMFetcher/1.0)", 7 Accept: "text/html", 8 }, 9 signal: AbortSignal.timeout(10000), 10 }); 11 12 if (!response.ok) return "Failed to fetch URL"; 13 14 const html = await response.text(); 15 const $ = cheerio.load(html); 16 17 $( 18 // Expanded list to include more non-content/boilerplate tags 19 "head, img, script, style, link, noscript, iframe, svg, nav, footer, header, form, input, button, select, option, label, canvas, figure, figcaption, object, embed, video, audio, source, track, picture, map, area, meta, base, col, colgroup, frame, frameset, param, dialog, template, menu, menuitem, output, progress", 20 ).remove(); 21 22 // Traverse body tag's children and extract text content 23 let cleanText = ""; 24 const body = $("body"); 25 26 function traverse(element: ReturnType<typeof $>) { 27 element.contents().each((_, node) => { 28 if (node.type === "text") { 29 const text = $(node).text().trim(); 30 if (text) { 31 cleanText += `${text} `; 32 } 33 } else if (node.type === "tag") { 34 const $node = $(node); 35 const tagName = node.name; 36 37 // Convert links to markdown format [Text](URL) 38 if (tagName === "a") { 39 const linkText = $node.text().trim(); 40 const linkHref = $node.attr("href"); 41 if (linkText && linkHref) { 42 try { 43 const absoluteUrl = new URL(linkHref, url).href; 44 cleanText += `[${linkText}](${absoluteUrl}) `; 45 } catch { 46 cleanText += `${linkText} `; 47 } 48 } 49 return; // Don't traverse into link children 50 } 51 52 // Add line breaks for block-level elements 53 if ( 54 ["p", "div", "h1", "h2", "h3", "h4", "h5", "h6", "li", "br"].includes( 55 tagName, 56 ) 57 ) { 58 traverse($node); 59 cleanText += "\n\n"; 60 } else { 61 traverse($node); 62 } 63 } 64 }); 65 } 66 67 traverse(body); 68 69 // 5. Final Cleanup: Normalize whitespace and remove excessive newlines 70 return cleanText 71 .replace(/[ \t]+/g, " ") // Collapse multiple spaces/tabs 72 .replace(/\n\s*\n/g, "\n\n") // Collapse multiple newlines 73 .trim(); 74}