556
Bn
bees-nest / files
bees-nest / src / app / api / crawl / result / route.ts
5.6 KB
1import { eq } from "drizzle-orm"; 2import { NextResponse } from "next/server"; 3import { z } from "zod"; 4import { domains, links, pages, urls } from "@/db/schema"; 5import { db } from "@/lib/db"; 6import { PRIORITIES } from "@/lib/queue"; 7 8const resultSchema = z.object({ 9 url: z.string().url(), 10 title: z.string().optional(), 11 description: z.string().optional(), 12 siteName: z.string().optional(), 13 image: z.string().optional(), 14 content: z.string().optional(), // Text content for search 15 newLinks: z.array(z.string().url()).optional(), 16 robotsTxt: z.string().optional(), // If the crawler found robots.txt 17}); 18 19export async function POST(req: Request) { 20 try { 21 // Basic Auth Check 22 const authHeader = req.headers.get("Authorization"); 23 if ( 24 process.env.API_SECRET && 25 authHeader !== `Bearer ${process.env.API_SECRET}` 26 ) { 27 return NextResponse.json({ error: "Unauthorized" }, { status: 401 }); 28 } 29 30 const body = await req.json(); 31 const data = resultSchema.parse(body); 32 33 // 1. Find the URL ID 34 const urlRecord = await db.query.urls.findFirst({ 35 where: eq(urls.url, data.url), 36 }); 37 38 if (!urlRecord) { 39 return NextResponse.json( 40 { error: "URL not found in queue" }, 41 { status: 404 }, 42 ); 43 } 44 45 await db.transaction(async (tx) => { 46 // 2. Mark URL as completed 47 await tx 48 .update(urls) 49 .set({ status: "completed", updatedAt: new Date() }) 50 .where(eq(urls.id, urlRecord.id)); 51 52 // 3. Save Page Data 53 const existingPage = await tx.query.pages.findFirst({ 54 where: eq(pages.urlId, urlRecord.id), 55 }); 56 57 if (existingPage) { 58 await tx 59 .update(pages) 60 .set({ 61 title: data.title, 62 description: data.description, 63 siteName: data.siteName, 64 image: data.image, 65 content: data.content, 66 }) 67 .where(eq(pages.id, existingPage.id)); 68 } else { 69 await tx.insert(pages).values({ 70 urlId: urlRecord.id, 71 title: data.title, 72 description: data.description, 73 siteName: data.siteName, 74 image: data.image, 75 content: data.content, 76 }); 77 } 78 79 // 4. Update robots.txt if provided 80 if (data.robotsTxt && urlRecord.domainId) { 81 await tx 82 .update(domains) 83 .set({ robotsTxt: data.robotsTxt, lastCrawledAt: new Date() }) 84 .where(eq(domains.id, urlRecord.domainId)); 85 } 86 }); 87 88 // 6. Post-transaction: Enqueue new links and create edges 89 if (data.newLinks && data.newLinks.length > 0) { 90 await processLinks(urlRecord.id, data.newLinks, urlRecord.depth + 1); 91 } 92 93 return NextResponse.json({ success: true }); 94 } catch (error) { 95 console.error("Save result error:", error); 96 return NextResponse.json({ error: "Processing error" }, { status: 500 }); 97 } 98} 99 100async function processLinks( 101 sourceUrlId: number, 102 linksList: string[], 103 depth: number, 104) { 105 if (depth > 3) return; 106 107 // Deduplicate links 108 const uniqueLinks = Array.from(new Set(linksList)); 109 if (uniqueLinks.length === 0) return; 110 111 // Map to domain names 112 const linkData = uniqueLinks 113 .map((link) => { 114 try { 115 const urlObj = new URL(link); 116 return { link, domain: urlObj.hostname }; 117 } catch (_e) { 118 return null; 119 } 120 }) 121 .filter((l): l is { link: string; domain: string } => l !== null); 122 123 if (linkData.length === 0) return; 124 125 const uniqueDomains = Array.from(new Set(linkData.map((l) => l.domain))); 126 127 // 1. Bulk Upsert Domains 128 // We need to upsert domains and get their IDs. 129 // Postgres doesn't easily return IDs on bulk onConflictDoUpdate unless we do it one by one or use CTEs (which Drizzle has partial support for). 130 // For simplicity and correctness with standard Drizzle, we will do sequential upserts for domains, 131 // BUT we will cache them in a map to avoid re-querying the same domain multiple times in this batch. 132 // Given standard crawler batches, uniqueDomains per page is small (< 50). 133 134 const domainIdMap = new Map<string, number>(); 135 136 // Parallelize domain upserts 137 await Promise.all( 138 uniqueDomains.map(async (domainName) => { 139 try { 140 const [d] = await db 141 .insert(domains) 142 .values({ domain: domainName }) 143 .onConflictDoUpdate({ 144 target: domains.domain, 145 set: { domain: domainName }, // no-op 146 }) 147 .returning({ id: domains.id }); 148 domainIdMap.set(domainName, d.id); 149 } catch (e) { 150 console.error(`Failed to process domain ${domainName}`, e); 151 } 152 }), 153 ); 154 155 // 2. Bulk Upsert URLs & 3. Create Edges 156 // Similarly, we want to add URLs. 157 // We can do this in parallel too. 158 159 await Promise.all( 160 linkData.map(async ({ link, domain }) => { 161 try { 162 const domainId = domainIdMap.get(domain); 163 if (!domainId) return; 164 165 // Upsert URL 166 // Note: We use onConflictDoUpdate to get the ID back reliably 167 const [urlRecord] = await db 168 .insert(urls) 169 .values({ 170 url: link, 171 domainId, 172 status: "pending", 173 priority: PRIORITIES.KNOWN_DOMAIN, // Simplified priority logic for bulk 174 depth, 175 }) 176 .onConflictDoUpdate({ 177 target: urls.url, 178 set: { url: link }, // no-op 179 }) 180 .returning({ id: urls.id }); 181 182 // Create Edge 183 await db 184 .insert(links) 185 .values({ 186 sourceUrlId, 187 targetUrlId: urlRecord.id, 188 }) 189 .onConflictDoNothing(); 190 } catch (_e) { 191 // Ignore 192 } 193 }), 194 ); 195}