xonecas.io
556
Bn

bees-nest / files

bees-nest / src / lib / queue.ts

1.7 KB
1import { domains, urls } from "@/db/schema";
2import { db } from "@/lib/db";
3
4export const PRIORITIES = {
5  NEW_DOMAIN: 100,
6  KNOWN_DOMAIN: 10,
7  SEED: 200,
8};
9
10export async function enqueueUrl(
11  url: string,
12  priority: number = PRIORITIES.KNOWN_DOMAIN,
13  depth: number = 0,
14) {
15  try {
16    const urlObj = new URL(url);
17    const domainName = urlObj.hostname;
18
19    // Upsert domain and get ID (atomic)
20    const [domainRecord] = await db
21      .insert(domains)
22      .values({ domain: domainName })
23      .onConflictDoUpdate({
24        target: domains.domain,
25        set: { domain: domainName }, // Dummy update to ensure returning works
26      })
27      .returning();
28
29    const domainId = domainRecord.id;
30
31    // Boost priority if it was just created (checked via created_at vs now? Or just assume)
32    // Actually, onConflictDoUpdate doesn't easily tell us if it was inserted or updated without a little trick.
33    // For now, we'll skip the "is new" check for priority boosting in this atomic version for simplicity,
34    // or we could check xmax system column if we really needed to.
35    // A simpler way: Check if createdAt is very recent.
36    const isNewDomain =
37      Date.now() - (domainRecord.createdAt?.getTime() ?? 0) < 1000;
38
39    if (isNewDomain) {
40      priority = Math.max(priority, PRIORITIES.NEW_DOMAIN);
41    }
42
43    // Insert URL if not exists
44    await db
45      .insert(urls)
46      .values({
47        url,
48        domainId,
49        priority,
50        depth,
51        status: "pending",
52      })
53      .onConflictDoNothing({ target: urls.url });
54
55    return { success: true };
56  } catch (error) {
57    console.error("Error enqueuing URL:", error);
58    return { success: false, error };
59  }
60}