xonecas.io
556
Bn

bees-nest / files

bees-nest / src / app / api / crawl / result / route.ts

5.6 KB
1import { eq } from "drizzle-orm";
2import { NextResponse } from "next/server";
3import { z } from "zod";
4import { domains, links, pages, urls } from "@/db/schema";
5import { db } from "@/lib/db";
6import { PRIORITIES } from "@/lib/queue";
7
8const resultSchema = z.object({
9  url: z.string().url(),
10  title: z.string().optional(),
11  description: z.string().optional(),
12  siteName: z.string().optional(),
13  image: z.string().optional(),
14  content: z.string().optional(), // Text content for search
15  newLinks: z.array(z.string().url()).optional(),
16  robotsTxt: z.string().optional(), // If the crawler found robots.txt
17});
18
19export async function POST(req: Request) {
20  try {
21    // Basic Auth Check
22    const authHeader = req.headers.get("Authorization");
23    if (
24      process.env.API_SECRET &&
25      authHeader !== `Bearer ${process.env.API_SECRET}`
26    ) {
27      return NextResponse.json({ error: "Unauthorized" }, { status: 401 });
28    }
29
30    const body = await req.json();
31    const data = resultSchema.parse(body);
32
33    // 1. Find the URL ID
34    const urlRecord = await db.query.urls.findFirst({
35      where: eq(urls.url, data.url),
36    });
37
38    if (!urlRecord) {
39      return NextResponse.json(
40        { error: "URL not found in queue" },
41        { status: 404 },
42      );
43    }
44
45    await db.transaction(async (tx) => {
46      // 2. Mark URL as completed
47      await tx
48        .update(urls)
49        .set({ status: "completed", updatedAt: new Date() })
50        .where(eq(urls.id, urlRecord.id));
51
52      // 3. Save Page Data
53      const existingPage = await tx.query.pages.findFirst({
54        where: eq(pages.urlId, urlRecord.id),
55      });
56
57      if (existingPage) {
58        await tx
59          .update(pages)
60          .set({
61            title: data.title,
62            description: data.description,
63            siteName: data.siteName,
64            image: data.image,
65            content: data.content,
66          })
67          .where(eq(pages.id, existingPage.id));
68      } else {
69        await tx.insert(pages).values({
70          urlId: urlRecord.id,
71          title: data.title,
72          description: data.description,
73          siteName: data.siteName,
74          image: data.image,
75          content: data.content,
76        });
77      }
78
79      // 4. Update robots.txt if provided
80      if (data.robotsTxt && urlRecord.domainId) {
81        await tx
82          .update(domains)
83          .set({ robotsTxt: data.robotsTxt, lastCrawledAt: new Date() })
84          .where(eq(domains.id, urlRecord.domainId));
85      }
86    });
87
88    // 6. Post-transaction: Enqueue new links and create edges
89    if (data.newLinks && data.newLinks.length > 0) {
90      await processLinks(urlRecord.id, data.newLinks, urlRecord.depth + 1);
91    }
92
93    return NextResponse.json({ success: true });
94  } catch (error) {
95    console.error("Save result error:", error);
96    return NextResponse.json({ error: "Processing error" }, { status: 500 });
97  }
98}
99
100async function processLinks(
101  sourceUrlId: number,
102  linksList: string[],
103  depth: number,
104) {
105  if (depth > 3) return;
106
107  // Deduplicate links
108  const uniqueLinks = Array.from(new Set(linksList));
109  if (uniqueLinks.length === 0) return;
110
111  // Map to domain names
112  const linkData = uniqueLinks
113    .map((link) => {
114      try {
115        const urlObj = new URL(link);
116        return { link, domain: urlObj.hostname };
117      } catch (_e) {
118        return null;
119      }
120    })
121    .filter((l): l is { link: string; domain: string } => l !== null);
122
123  if (linkData.length === 0) return;
124
125  const uniqueDomains = Array.from(new Set(linkData.map((l) => l.domain)));
126
127  // 1. Bulk Upsert Domains
128  // We need to upsert domains and get their IDs.
129  // Postgres doesn't easily return IDs on bulk onConflictDoUpdate unless we do it one by one or use CTEs (which Drizzle has partial support for).
130  // For simplicity and correctness with standard Drizzle, we will do sequential upserts for domains,
131  // BUT we will cache them in a map to avoid re-querying the same domain multiple times in this batch.
132  // Given standard crawler batches, uniqueDomains per page is small (< 50).
133
134  const domainIdMap = new Map<string, number>();
135
136  // Parallelize domain upserts
137  await Promise.all(
138    uniqueDomains.map(async (domainName) => {
139      try {
140        const [d] = await db
141          .insert(domains)
142          .values({ domain: domainName })
143          .onConflictDoUpdate({
144            target: domains.domain,
145            set: { domain: domainName }, // no-op
146          })
147          .returning({ id: domains.id });
148        domainIdMap.set(domainName, d.id);
149      } catch (e) {
150        console.error(`Failed to process domain ${domainName}`, e);
151      }
152    }),
153  );
154
155  // 2. Bulk Upsert URLs & 3. Create Edges
156  // Similarly, we want to add URLs.
157  // We can do this in parallel too.
158
159  await Promise.all(
160    linkData.map(async ({ link, domain }) => {
161      try {
162        const domainId = domainIdMap.get(domain);
163        if (!domainId) return;
164
165        // Upsert URL
166        // Note: We use onConflictDoUpdate to get the ID back reliably
167        const [urlRecord] = await db
168          .insert(urls)
169          .values({
170            url: link,
171            domainId,
172            status: "pending",
173            priority: PRIORITIES.KNOWN_DOMAIN, // Simplified priority logic for bulk
174            depth,
175          })
176          .onConflictDoUpdate({
177            target: urls.url,
178            set: { url: link }, // no-op
179          })
180          .returning({ id: urls.id });
181
182        // Create Edge
183        await db
184          .insert(links)
185          .values({
186            sourceUrlId,
187            targetUrlId: urlRecord.id,
188          })
189          .onConflictDoNothing();
190      } catch (_e) {
191        // Ignore
192      }
193    }),
194  );
195}