556
Bn
bees-nest / files
bees-nest / src / lib / queue.ts
1.7 KB
1import { domains, urls } from "@/db/schema"; 2import { db } from "@/lib/db"; 3 4export const PRIORITIES = { 5 NEW_DOMAIN: 100, 6 KNOWN_DOMAIN: 10, 7 SEED: 200, 8}; 9 10export async function enqueueUrl( 11 url: string, 12 priority: number = PRIORITIES.KNOWN_DOMAIN, 13 depth: number = 0, 14) { 15 try { 16 const urlObj = new URL(url); 17 const domainName = urlObj.hostname; 18 19 // Upsert domain and get ID (atomic) 20 const [domainRecord] = await db 21 .insert(domains) 22 .values({ domain: domainName }) 23 .onConflictDoUpdate({ 24 target: domains.domain, 25 set: { domain: domainName }, // Dummy update to ensure returning works 26 }) 27 .returning(); 28 29 const domainId = domainRecord.id; 30 31 // Boost priority if it was just created (checked via created_at vs now? Or just assume) 32 // Actually, onConflictDoUpdate doesn't easily tell us if it was inserted or updated without a little trick. 33 // For now, we'll skip the "is new" check for priority boosting in this atomic version for simplicity, 34 // or we could check xmax system column if we really needed to. 35 // A simpler way: Check if createdAt is very recent. 36 const isNewDomain = 37 Date.now() - (domainRecord.createdAt?.getTime() ?? 0) < 1000; 38 39 if (isNewDomain) { 40 priority = Math.max(priority, PRIORITIES.NEW_DOMAIN); 41 } 42 43 // Insert URL if not exists 44 await db 45 .insert(urls) 46 .values({ 47 url, 48 domainId, 49 priority, 50 depth, 51 status: "pending", 52 }) 53 .onConflictDoNothing({ target: urls.url }); 54 55 return { success: true }; 56 } catch (error) { 57 console.error("Error enqueuing URL:", error); 58 return { success: false, error }; 59 } 60}