xonecas.io
5414
Sa

sat / commit 419b5f3

feat: simple fetch page tool

xonecas2026-01-16 419b5f311a299bdf56b4391108ac48e130c38385 parent 56e4f29

8 files changed

package-lock.json+304−0
@@ -11,6 +11,7 @@
1111 "@base-ui/react": "^1.0.0",
1212 "@langchain/core": "^1.1.13",
1313 "@langchain/ollama": "^1.2.0",
14+ "cheerio": "^1.1.2",
1415 "class-variance-authority": "^0.7.1",
1516 "clsx": "^2.1.1",
1617 "drizzle-orm": "^0.45.1",
@@ -5001,6 +5002,12 @@
50015002 "url": "https://opencollective.com/express"
50025003 }
50035004 },
5005+ "node_modules/boolbase": {
5006+ "version": "1.0.0",
5007+ "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz",
5008+ "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==",
5009+ "license": "ISC"
5010+ },
50045011 "node_modules/braces": {
50055012 "version": "3.0.3",
50065013 "resolved": "https://registry.npmjs.org/braces/-/braces-3.0.3.tgz",
@@ -5209,6 +5216,48 @@
52095216 "url": "https://github.com/sponsors/wooorm"
52105217 }
52115218 },
5219+ "node_modules/cheerio": {
5220+ "version": "1.1.2",
5221+ "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.1.2.tgz",
5222+ "integrity": "sha512-IkxPpb5rS/d1IiLbHMgfPuS0FgiWTtFIm/Nj+2woXDLTZ7fOT2eqzgYbdMlLweqlHbsZjxEChoVK+7iph7jyQg==",
5223+ "license": "MIT",
5224+ "dependencies": {
5225+ "cheerio-select": "^2.1.0",
5226+ "dom-serializer": "^2.0.0",
5227+ "domhandler": "^5.0.3",
5228+ "domutils": "^3.2.2",
5229+ "encoding-sniffer": "^0.2.1",
5230+ "htmlparser2": "^10.0.0",
5231+ "parse5": "^7.3.0",
5232+ "parse5-htmlparser2-tree-adapter": "^7.1.0",
5233+ "parse5-parser-stream": "^7.1.2",
5234+ "undici": "^7.12.0",
5235+ "whatwg-mimetype": "^4.0.0"
5236+ },
5237+ "engines": {
5238+ "node": ">=20.18.1"
5239+ },
5240+ "funding": {
5241+ "url": "https://github.com/cheeriojs/cheerio?sponsor=1"
5242+ }
5243+ },
5244+ "node_modules/cheerio-select": {
5245+ "version": "2.1.0",
5246+ "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz",
5247+ "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==",
5248+ "license": "BSD-2-Clause",
5249+ "dependencies": {
5250+ "boolbase": "^1.0.0",
5251+ "css-select": "^5.1.0",
5252+ "css-what": "^6.1.0",
5253+ "domelementtype": "^2.3.0",
5254+ "domhandler": "^5.0.3",
5255+ "domutils": "^3.0.1"
5256+ },
5257+ "funding": {
5258+ "url": "https://github.com/sponsors/fb55"
5259+ }
5260+ },
52125261 "node_modules/class-variance-authority": {
52135262 "version": "0.7.1",
52145263 "resolved": "https://registry.npmjs.org/class-variance-authority/-/class-variance-authority-0.7.1.tgz",
@@ -5516,6 +5565,34 @@
55165565 "node": ">= 8"
55175566 }
55185567 },
5568+ "node_modules/css-select": {
5569+ "version": "5.2.2",
5570+ "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.2.2.tgz",
5571+ "integrity": "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==",
5572+ "license": "BSD-2-Clause",
5573+ "dependencies": {
5574+ "boolbase": "^1.0.0",
5575+ "css-what": "^6.1.0",
5576+ "domhandler": "^5.0.2",
5577+ "domutils": "^3.0.1",
5578+ "nth-check": "^2.0.1"
5579+ },
5580+ "funding": {
5581+ "url": "https://github.com/sponsors/fb55"
5582+ }
5583+ },
5584+ "node_modules/css-what": {
5585+ "version": "6.2.2",
5586+ "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz",
5587+ "integrity": "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==",
5588+ "license": "BSD-2-Clause",
5589+ "engines": {
5590+ "node": ">= 6"
5591+ },
5592+ "funding": {
5593+ "url": "https://github.com/sponsors/fb55"
5594+ }
5595+ },
55195596 "node_modules/cssesc": {
55205597 "version": "3.0.0",
55215598 "resolved": "https://registry.npmjs.org/cssesc/-/cssesc-3.0.0.tgz",
@@ -5701,6 +5778,61 @@
57015778 "node": ">=0.3.1"
57025779 }
57035780 },
5781+ "node_modules/dom-serializer": {
5782+ "version": "2.0.0",
5783+ "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz",
5784+ "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==",
5785+ "license": "MIT",
5786+ "dependencies": {
5787+ "domelementtype": "^2.3.0",
5788+ "domhandler": "^5.0.2",
5789+ "entities": "^4.2.0"
5790+ },
5791+ "funding": {
5792+ "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1"
5793+ }
5794+ },
5795+ "node_modules/domelementtype": {
5796+ "version": "2.3.0",
5797+ "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz",
5798+ "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==",
5799+ "funding": [
5800+ {
5801+ "type": "github",
5802+ "url": "https://github.com/sponsors/fb55"
5803+ }
5804+ ],
5805+ "license": "BSD-2-Clause"
5806+ },
5807+ "node_modules/domhandler": {
5808+ "version": "5.0.3",
5809+ "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz",
5810+ "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==",
5811+ "license": "BSD-2-Clause",
5812+ "dependencies": {
5813+ "domelementtype": "^2.3.0"
5814+ },
5815+ "engines": {
5816+ "node": ">= 4"
5817+ },
5818+ "funding": {
5819+ "url": "https://github.com/fb55/domhandler?sponsor=1"
5820+ }
5821+ },
5822+ "node_modules/domutils": {
5823+ "version": "3.2.2",
5824+ "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz",
5825+ "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==",
5826+ "license": "BSD-2-Clause",
5827+ "dependencies": {
5828+ "dom-serializer": "^2.0.0",
5829+ "domelementtype": "^2.3.0",
5830+ "domhandler": "^5.0.3"
5831+ },
5832+ "funding": {
5833+ "url": "https://github.com/fb55/domutils?sponsor=1"
5834+ }
5835+ },
57045836 "node_modules/dotenv": {
57055837 "version": "17.2.3",
57065838 "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz",
@@ -5912,6 +6044,31 @@
59126044 "node": ">= 0.8"
59136045 }
59146046 },
6047+ "node_modules/encoding-sniffer": {
6048+ "version": "0.2.1",
6049+ "resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.1.tgz",
6050+ "integrity": "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw==",
6051+ "license": "MIT",
6052+ "dependencies": {
6053+ "iconv-lite": "^0.6.3",
6054+ "whatwg-encoding": "^3.1.1"
6055+ },
6056+ "funding": {
6057+ "url": "https://github.com/fb55/encoding-sniffer?sponsor=1"
6058+ }
6059+ },
6060+ "node_modules/encoding-sniffer/node_modules/iconv-lite": {
6061+ "version": "0.6.3",
6062+ "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
6063+ "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
6064+ "license": "MIT",
6065+ "dependencies": {
6066+ "safer-buffer": ">= 2.1.2 < 3.0.0"
6067+ },
6068+ "engines": {
6069+ "node": ">=0.10.0"
6070+ }
6071+ },
59156072 "node_modules/enhanced-resolve": {
59166073 "version": "5.18.4",
59176074 "resolved": "https://registry.npmjs.org/enhanced-resolve/-/enhanced-resolve-5.18.4.tgz",
@@ -5926,6 +6083,18 @@
59266083 "node": ">=10.13.0"
59276084 }
59286085 },
6086+ "node_modules/entities": {
6087+ "version": "4.5.0",
6088+ "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz",
6089+ "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==",
6090+ "license": "BSD-2-Clause",
6091+ "engines": {
6092+ "node": ">=0.12"
6093+ },
6094+ "funding": {
6095+ "url": "https://github.com/fb55/entities?sponsor=1"
6096+ }
6097+ },
59296098 "node_modules/env-paths": {
59306099 "version": "2.2.1",
59316100 "resolved": "https://registry.npmjs.org/env-paths/-/env-paths-2.2.1.tgz",
@@ -6648,6 +6817,37 @@
66486817 "url": "https://opencollective.com/unified"
66496818 }
66506819 },
6820+ "node_modules/htmlparser2": {
6821+ "version": "10.0.0",
6822+ "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.0.0.tgz",
6823+ "integrity": "sha512-TwAZM+zE5Tq3lrEHvOlvwgj1XLWQCtaaibSN11Q+gGBAS7Y1uZSWwXXRe4iF6OXnaq1riyQAPFOBtYc77Mxq0g==",
6824+ "funding": [
6825+ "https://github.com/fb55/htmlparser2?sponsor=1",
6826+ {
6827+ "type": "github",
6828+ "url": "https://github.com/sponsors/fb55"
6829+ }
6830+ ],
6831+ "license": "MIT",
6832+ "dependencies": {
6833+ "domelementtype": "^2.3.0",
6834+ "domhandler": "^5.0.3",
6835+ "domutils": "^3.2.1",
6836+ "entities": "^6.0.0"
6837+ }
6838+ },
6839+ "node_modules/htmlparser2/node_modules/entities": {
6840+ "version": "6.0.1",
6841+ "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz",
6842+ "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==",
6843+ "license": "BSD-2-Clause",
6844+ "engines": {
6845+ "node": ">=0.12"
6846+ },
6847+ "funding": {
6848+ "url": "https://github.com/fb55/entities?sponsor=1"
6849+ }
6850+ },
66516851 "node_modules/http-errors": {
66526852 "version": "2.0.1",
66536853 "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.1.tgz",
@@ -8523,6 +8723,18 @@
85238723 "url": "https://github.com/sponsors/sindresorhus"
85248724 }
85258725 },
8726+ "node_modules/nth-check": {
8727+ "version": "2.1.1",
8728+ "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz",
8729+ "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==",
8730+ "license": "BSD-2-Clause",
8731+ "dependencies": {
8732+ "boolbase": "^1.0.0"
8733+ },
8734+ "funding": {
8735+ "url": "https://github.com/fb55/nth-check?sponsor=1"
8736+ }
8737+ },
85268738 "node_modules/object-assign": {
85278739 "version": "4.1.1",
85288740 "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
@@ -8772,6 +8984,55 @@
87728984 "url": "https://github.com/sponsors/sindresorhus"
87738985 }
87748986 },
8987+ "node_modules/parse5": {
8988+ "version": "7.3.0",
8989+ "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz",
8990+ "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==",
8991+ "license": "MIT",
8992+ "dependencies": {
8993+ "entities": "^6.0.0"
8994+ },
8995+ "funding": {
8996+ "url": "https://github.com/inikulin/parse5?sponsor=1"
8997+ }
8998+ },
8999+ "node_modules/parse5-htmlparser2-tree-adapter": {
9000+ "version": "7.1.0",
9001+ "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz",
9002+ "integrity": "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==",
9003+ "license": "MIT",
9004+ "dependencies": {
9005+ "domhandler": "^5.0.3",
9006+ "parse5": "^7.0.0"
9007+ },
9008+ "funding": {
9009+ "url": "https://github.com/inikulin/parse5?sponsor=1"
9010+ }
9011+ },
9012+ "node_modules/parse5-parser-stream": {
9013+ "version": "7.1.2",
9014+ "resolved": "https://registry.npmjs.org/parse5-parser-stream/-/parse5-parser-stream-7.1.2.tgz",
9015+ "integrity": "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==",
9016+ "license": "MIT",
9017+ "dependencies": {
9018+ "parse5": "^7.0.0"
9019+ },
9020+ "funding": {
9021+ "url": "https://github.com/inikulin/parse5?sponsor=1"
9022+ }
9023+ },
9024+ "node_modules/parse5/node_modules/entities": {
9025+ "version": "6.0.1",
9026+ "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz",
9027+ "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==",
9028+ "license": "BSD-2-Clause",
9029+ "engines": {
9030+ "node": ">=0.12"
9031+ },
9032+ "funding": {
9033+ "url": "https://github.com/fb55/entities?sponsor=1"
9034+ }
9035+ },
87759036 "node_modules/parseurl": {
87769037 "version": "1.3.3",
87779038 "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
@@ -10238,6 +10499,15 @@
1023810499 "node": ">=14.17"
1023910500 }
1024010501 },
10502+ "node_modules/undici": {
10503+ "version": "7.18.2",
10504+ "resolved": "https://registry.npmjs.org/undici/-/undici-7.18.2.tgz",
10505+ "integrity": "sha512-y+8YjDFzWdQlSE9N5nzKMT3g4a5UBX1HKowfdXh0uvAnTaqqwqB92Jt4UXBAeKekDs5IaDKyJFR4X1gYVCgXcw==",
10506+ "license": "MIT",
10507+ "engines": {
10508+ "node": ">=20.18.1"
10509+ }
10510+ },
1024110511 "node_modules/undici-types": {
1024210512 "version": "6.21.0",
1024310513 "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-6.21.0.tgz",
@@ -10527,12 +10797,46 @@
1052710797 "node": ">= 8"
1052810798 }
1052910799 },
10800+ "node_modules/whatwg-encoding": {
10801+ "version": "3.1.1",
10802+ "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz",
10803+ "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==",
10804+ "deprecated": "Use @exodus/bytes instead for a more spec-conformant and faster implementation",
10805+ "license": "MIT",
10806+ "dependencies": {
10807+ "iconv-lite": "0.6.3"
10808+ },
10809+ "engines": {
10810+ "node": ">=18"
10811+ }
10812+ },
10813+ "node_modules/whatwg-encoding/node_modules/iconv-lite": {
10814+ "version": "0.6.3",
10815+ "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz",
10816+ "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==",
10817+ "license": "MIT",
10818+ "dependencies": {
10819+ "safer-buffer": ">= 2.1.2 < 3.0.0"
10820+ },
10821+ "engines": {
10822+ "node": ">=0.10.0"
10823+ }
10824+ },
1053010825 "node_modules/whatwg-fetch": {
1053110826 "version": "3.6.20",
1053210827 "resolved": "https://registry.npmjs.org/whatwg-fetch/-/whatwg-fetch-3.6.20.tgz",
1053310828 "integrity": "sha512-EqhiFU6daOA8kpjOWTL0olhVOF3i7OrFzSYiGsEMB8GcXS+RrzauAERX65xMeNWVqxA6HXH2m69Z9LaKKdisfg==",
1053410829 "license": "MIT"
1053510830 },
10831+ "node_modules/whatwg-mimetype": {
10832+ "version": "4.0.0",
10833+ "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz",
10834+ "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==",
10835+ "license": "MIT",
10836+ "engines": {
10837+ "node": ">=18"
10838+ }
10839+ },
1053610840 "node_modules/which": {
1053710841 "version": "4.0.0",
1053810842 "resolved": "https://registry.npmjs.org/which/-/which-4.0.0.tgz",
package.json+1−0
@@ -16,6 +16,7 @@
1616 "@base-ui/react": "^1.0.0",
1717 "@langchain/core": "^1.1.13",
1818 "@langchain/ollama": "^1.2.0",
19+ "cheerio": "^1.1.2",
1920 "class-variance-authority": "^0.7.1",
2021 "clsx": "^2.1.1",
2122 "drizzle-orm": "^0.45.1",
src/app/api/stream/route.ts+10−20
@@ -1,6 +1,8 @@
1+import { asc, eq } from "drizzle-orm";
12 import { NextResponse } from "next/server";
2-import { type ChatHistory, chat, type StreamEvent } from "@/lib/chat";
3-import { summarizeIfNeeded } from "@/lib/summarize";
3+import { db } from "@/db";
4+import { type Turn, type TurnWithToolCalls, turns } from "@/db/schema";
5+import { chat, type StreamEvent } from "@/lib/chat";
46
57 export async function POST(request: Request) {
68 const { userInput, conversationId } = await request.json();
@@ -8,25 +10,13 @@ export async function POST(request: Request) {
810 const encoder = new TextEncoder();
911 const stream = new ReadableStream({
1012 async start(controller) {
11- let history: ChatHistory | undefined;
13+ let history: Turn[] | undefined;
1214
1315 if (conversationId) {
14- try {
15- const result = await summarizeIfNeeded(conversationId);
16- history = result.history;
17-
18- // Notify client if summarization occurred
19- if (result.summarizedCount > 0) {
20- controller.enqueue(
21- encoder.encode(
22- `${JSON.stringify({ type: "summarized", messageCount: result.summarizedCount })}\n`,
23- ),
24- );
25- }
26- } catch (error) {
27- console.error("Failed to fetch conversation history:", error);
28- // Continue without history if fetch fails
29- }
16+ history = await db.query.turns.findMany({
17+ where: eq(turns.conversationId, conversationId),
18+ orderBy: [asc(turns.createdAt)],
19+ });
3020 }
3121
3222 await chat(
@@ -35,7 +25,7 @@ export async function POST(request: Request) {
3525 // Use newline-delimited JSON for easier parsing
3626 controller.enqueue(encoder.encode(`${JSON.stringify(event)}\n`));
3727 },
38- history,
28+ history as TurnWithToolCalls[],
3929 conversationId,
4030 );
4131
src/db/schema.ts+12−0
@@ -40,3 +40,15 @@ export type Conversation = typeof conversations.$inferSelect;
4040 export type NewConversation = typeof conversations.$inferInsert;
4141 export type Turn = typeof turns.$inferSelect;
4242 export type NewTurn = typeof turns.$inferInsert;
43+
44+// Types for chat history
45+interface ToolCallData {
46+ id: string;
47+ name: string;
48+ args: Record<string, unknown>;
49+ result?: string;
50+}
51+
52+export interface TurnWithToolCalls extends Turn {
53+ toolCalls: ToolCallData[];
54+}
src/lib/agent.ts+85−0
@@ -0,0 +1,85 @@
1+import { tool } from "@langchain/core/tools";
2+import { MemorySaver } from "@langchain/langgraph";
3+import { ChatOllama } from "@langchain/ollama";
4+import { createAgent, SystemMessage, summarizationMiddleware } from "langchain";
5+import { z } from "zod";
6+import { fetchURLContents } from "@/lib/fetchURLContents";
7+
8+interface SearchResult {
9+ url: string;
10+ title: string;
11+ content: string;
12+ score: number;
13+}
14+
15+const searchWeb = tool(
16+ async ({ query }: { query: string }) => {
17+ const response = await fetch(
18+ `http://localhost:8080/search?q=${query}&format=json`,
19+ );
20+ const data = await response.json();
21+ const plainTextResults = data.results
22+ .map((result: SearchResult) => {
23+ console.log(
24+ `result: ${result.title}: ${result.content}\n[score: ${result.score}] [source: ${result.url}]`,
25+ );
26+ return `${result.title}: ${result.content}\n[score: ${result.score}] [source: ${result.url}]`;
27+ })
28+ .join("\n\n");
29+ return plainTextResults;
30+ },
31+ {
32+ name: "search_web",
33+ description: "search the web for a specified query string",
34+ schema: z.object({
35+ query: z.string().describe("Search terms to look for"),
36+ }),
37+ },
38+);
39+
40+const fetchPageContents = tool(
41+ async ({ url }: { url: string }) => {
42+ const contents = await fetchURLContents(url);
43+ return contents;
44+ },
45+ {
46+ name: "fetch_page_contents",
47+ description: "fetch the contents of a web page",
48+ schema: z.object({ url: z.string().describe("The URL to fetch") }),
49+ },
50+);
51+
52+// TODO: https://docs.langchain.com/oss/javascript/langchain/short-term-memory#in-production
53+const checkpointer = new MemorySaver();
54+
55+// Singleton agent instance
56+let agentInstance: ReturnType<typeof createAgent> | null = null;
57+
58+export function getAgent() {
59+ if (!agentInstance) {
60+ const llm = new ChatOllama({
61+ model: "qwen3:8b",
62+ });
63+
64+ agentInstance = createAgent({
65+ model: llm,
66+ tools: [searchWeb, fetchPageContents],
67+ middleware: [
68+ summarizationMiddleware({
69+ model: llm,
70+ trigger: { tokens: 4000 },
71+ keep: { messages: 10 },
72+ }),
73+ ],
74+ checkpointer,
75+ systemPrompt: new SystemMessage(
76+ `You are a helpful web search assistant. Always respond in English, regardless of the language used in search results or user queries.
77+You have access to a tool that searches the web, and a tool that fetches the contents of a web page.
78+Create optimized search queries from the user's input and use them with the search_web tool to get the best results.
79+Use the fetch_page_contents tool to fetch the contents of a web page from the results of the search_web tool if the user's query is about the content of a web page.
80+Review the results and provide a concise summary of the information found in English. Include sources and links if available.`,
81+ ),
82+ });
83+ }
84+ return agentInstance;
85+}
src/lib/chat.ts+11−115
@@ -3,40 +3,11 @@ import {
33 AIMessageChunk,
44 type BaseMessage,
55 HumanMessage,
6- SystemMessage as LangChainSystemMessage,
76 ToolMessage,
87 } from "@langchain/core/messages";
9-import { tool } from "@langchain/core/tools";
108 import { concat } from "@langchain/core/utils/stream";
11-import { MemorySaver } from "@langchain/langgraph";
12-import { ChatOllama } from "@langchain/ollama";
13-import { createAgent, SystemMessage } from "langchain";
14-import { z } from "zod";
15-
16-// Relevant documentation:
17-//
18-// https://v03.api.js.langchain.com/classes/_langchain_ollama.ChatOllama.html
19-// https://docs.langchain.com/oss/javascript/integrations/chat/ollama
20-
21-const searchWeb = tool(
22- async ({ query }: { query: string }) => {
23- console.log(`searching for ${query}`);
24-
25- const response = await fetch(
26- `http://localhost:8080/search?q=${query}&format=json`,
27- );
28- const data = await response.json();
29- // TODO: create a plain text version of the results for the llm
30- return JSON.stringify(data.results ?? [], null, 2);
31- },
32- {
33- name: "search_web",
34- description: "search the web for a specified query string",
35- schema: z.object({
36- query: z.string().describe("Search terms to look for"),
37- }),
38- },
39-);
9+import type { TurnWithToolCalls } from "@/db/schema";
10+import { getAgent } from "@/lib/agent";
4011
4112 export type StreamEvent =
4213 | { type: "token"; content: string }
@@ -47,72 +18,12 @@ export type StreamEvent =
4718 name: string;
4819 args: Record<string, unknown>;
4920 }
50- | { type: "tool_result"; id: string; result: string }
51- | { type: "summarized"; messageCount: number };
52-
53-// Types for chat history
54-export interface ToolCallData {
55- id: string;
56- name: string;
57- args: Record<string, unknown>;
58- result?: string;
59-}
60-
61-export interface ChatHistoryTurn {
62- role: "user" | "assistant";
63- content: string;
64- toolCalls?: ToolCallData[];
65-}
66-
67-export interface ChatHistory {
68- summary?: string;
69- turns: ChatHistoryTurn[];
70-}
71-
72-// Singleton checkpointer - persists in memory across requests
73-// Note: Lost on server restart, but we re-populate from PostgreSQL
74-const checkpointer = new MemorySaver();
75-
76-// Singleton agent instance
77-let agentInstance: ReturnType<typeof createAgent> | null = null;
21+ | { type: "tool_result"; id: string; result: string };
7822
79-function getAgent() {
80- if (!agentInstance) {
81- const llm = new ChatOllama({
82- model: "qwen3:8b",
83- });
84-
85- agentInstance = createAgent({
86- model: llm,
87- tools: [searchWeb],
88- checkpointer,
89- systemPrompt: new SystemMessage(
90- "You are a helpful web search assistant. Always respond in English, regardless of the language used in search results or user queries." +
91- " You have access to a tool that searches the web." +
92- " Create optimized search queries from the user's input, use them with the search_web tool to get the best results." +
93- " Review the results and provide a concise summary of the information found in English. Include sources and links if available.",
94- ),
95- });
96- }
97- return agentInstance;
98-}
99-
100-/**
101- * Convert history turns to LangChain BaseMessage array
102- */
103-function historyToMessages(history: ChatHistory): BaseMessage[] {
23+function historyToMessages(history: TurnWithToolCalls[]): BaseMessage[] {
10424 const messages: BaseMessage[] = [];
10525
106- // Prepend summary as system context if exists
107- if (history.summary) {
108- messages.push(
109- new LangChainSystemMessage(
110- `Previous conversation summary:\n${history.summary}`,
111- ),
112- );
113- }
114-
115- for (const turn of history.turns) {
26+ for (const turn of history) {
11627 if (turn.role === "user") {
11728 messages.push(new HumanMessage(turn.content));
11829 } else {
@@ -150,7 +61,7 @@ function historyToMessages(history: ChatHistory): BaseMessage[] {
15061 export async function chat(
15162 userInput: string,
15263 onEvent: (event: StreamEvent) => void,
153- history?: ChatHistory,
64+ history?: TurnWithToolCalls[],
15465 conversationId?: string,
15566 ) {
15667 const agent = getAgent();
@@ -159,7 +70,7 @@ export async function chat(
15970
16071 // Check if this thread has existing state in the checkpointer
16172 // If not, we need to populate it from PostgreSQL history
162- if (history && history.turns.length > 0) {
73+ if (history && history.length > 0) {
16374 try {
16475 const existingState = (await agent.getState(config)) as {
16576 values?: { messages?: unknown[] };
@@ -173,18 +84,9 @@ export async function chat(
17384 stateValues.messages.length > 0;
17485
17586 if (!hasMessages) {
176- // Convert history to messages (excluding the current user message if it's already there)
177- const lastTurn = history.turns.at(-1);
178- const turnsToLoad =
179- lastTurn?.role === "user" && lastTurn.content === userInput
180- ? { ...history, turns: history.turns.slice(0, -1) }
181- : history;
182-
183- if (turnsToLoad.turns.length > 0) {
184- const historyMessages = historyToMessages(turnsToLoad);
185- // Use updateState to populate the checkpointer
186- await agent.updateState(config, { messages: historyMessages });
187- }
87+ const historyMessages = historyToMessages(history);
88+ // Use updateState to populate the checkpointer
89+ await agent.updateState(config, { messages: historyMessages });
18890 }
18991 } catch (error) {
19092 console.error("[chat] Error checking/populating state:", error);
@@ -214,12 +116,6 @@ export async function chat(
214116 onEvent({ type: "token", content: message.content });
215117 }
216118
217- // Stream reasoning tokens immediately
218- const reasoning = message.additional_kwargs?.reasoning_content;
219- if (typeof reasoning === "string" && reasoning) {
220- onEvent({ type: "reasoning", content: reasoning });
221- }
222-
223119 // Aggregate for tool call detection within messages mode
224120 aggregatedAiChunk =
225121 aggregatedAiChunk === undefined
@@ -228,7 +124,7 @@ export async function chat(
228124 }
229125 }
230126
231- // Handle state updates from "updates" mode (tool calls and results)
127+ // Handle state updates from "updates" mode (tool calls and reasoning)
232128 if (mode === "updates") {
233129 const update = chunk as Record<string, { messages?: unknown[] }>;
234130
src/lib/fetchURLContents.ts+74−0
@@ -0,0 +1,74 @@
1+import * as cheerio from "cheerio";
2+
3+export async function fetchURLContents(url: string): Promise<string> {
4+ const response = await fetch(url, {
5+ headers: {
6+ "User-Agent": "Mozilla/5.0 (compatible; LLMFetcher/1.0)",
7+ Accept: "text/html",
8+ },
9+ signal: AbortSignal.timeout(10000),
10+ });
11+
12+ if (!response.ok) return "Failed to fetch URL";
13+
14+ const html = await response.text();
15+ const $ = cheerio.load(html);
16+
17+ $(
18+ // Expanded list to include more non-content/boilerplate tags
19+ "head, img, script, style, link, noscript, iframe, svg, nav, footer, header, form, input, button, select, option, label, canvas, figure, figcaption, object, embed, video, audio, source, track, picture, map, area, meta, base, col, colgroup, frame, frameset, param, dialog, template, menu, menuitem, output, progress",
20+ ).remove();
21+
22+ // Traverse body tag's children and extract text content
23+ let cleanText = "";
24+ const body = $("body");
25+
26+ function traverse(element: ReturnType<typeof $>) {
27+ element.contents().each((_, node) => {
28+ if (node.type === "text") {
29+ const text = $(node).text().trim();
30+ if (text) {
31+ cleanText += `${text} `;
32+ }
33+ } else if (node.type === "tag") {
34+ const $node = $(node);
35+ const tagName = node.name;
36+
37+ // Convert links to markdown format [Text](URL)
38+ if (tagName === "a") {
39+ const linkText = $node.text().trim();
40+ const linkHref = $node.attr("href");
41+ if (linkText && linkHref) {
42+ try {
43+ const absoluteUrl = new URL(linkHref, url).href;
44+ cleanText += `[${linkText}](${absoluteUrl}) `;
45+ } catch {
46+ cleanText += `${linkText} `;
47+ }
48+ }
49+ return; // Don't traverse into link children
50+ }
51+
52+ // Add line breaks for block-level elements
53+ if (
54+ ["p", "div", "h1", "h2", "h3", "h4", "h5", "h6", "li", "br"].includes(
55+ tagName,
56+ )
57+ ) {
58+ traverse($node);
59+ cleanText += "\n\n";
60+ } else {
61+ traverse($node);
62+ }
63+ }
64+ });
65+ }
66+
67+ traverse(body);
68+
69+ // 5. Final Cleanup: Normalize whitespace and remove excessive newlines
70+ return cleanText
71+ .replace(/[ \t]+/g, " ") // Collapse multiple spaces/tabs
72+ .replace(/\n\s*\n/g, "\n\n") // Collapse multiple newlines
73+ .trim();
74+}
src/lib/summarize.ts+0−103
@@ -1,103 +0,0 @@
1-import { ChatOllama } from "@langchain/ollama";
2-import {
3- getConversation,
4- updateConversationSummary,
5-} from "@/app/actions/conversations";
6-import type { Turn } from "@/db/schema";
7-import type { ChatHistory, ChatHistoryTurn, ToolCallData } from "@/lib/chat";
8-
9-const TOKEN_THRESHOLD = 24000; // ~75% of qwen3:8b's 32k context
10-const MESSAGES_TO_KEEP = 10;
11-
12-export interface SummarizeResult {
13- history: ChatHistory;
14- summarizedCount: number; // Number of messages that were summarized (0 if none)
15-}
16-
17-/**
18- * Fetches conversation history and summarizes older messages if needed.
19- * Returns the history ready to be passed to the chat function.
20- */
21-export async function summarizeIfNeeded(
22- conversationId: string,
23-): Promise<SummarizeResult> {
24- const conversation = await getConversation(conversationId);
25- if (!conversation) {
26- throw new Error("Conversation not found");
27- }
28-
29- const turns = conversation.turns;
30- const estimatedTokens = estimateTokens(turns);
31-
32- if (estimatedTokens > TOKEN_THRESHOLD && turns.length > MESSAGES_TO_KEEP) {
33- const toSummarize = turns.slice(0, -MESSAGES_TO_KEEP);
34- const toKeep = turns.slice(-MESSAGES_TO_KEEP);
35-
36- const summaryText = await generateSummary(
37- conversation.summary,
38- toSummarize,
39- );
40-
41- await updateConversationSummary(conversationId, summaryText);
42-
43- return {
44- history: {
45- summary: summaryText,
46- turns: toKeep.map(turnToHistoryTurn),
47- },
48- summarizedCount: toSummarize.length,
49- };
50- }
51-
52- return {
53- history: {
54- summary: conversation.summary ?? undefined,
55- turns: turns.map(turnToHistoryTurn),
56- },
57- summarizedCount: 0,
58- };
59-}
60-
61-/**
62- * Rough token estimation: ~4 characters per token
63- */
64-function estimateTokens(turns: Turn[]): number {
65- return turns.reduce((sum, t) => sum + t.content.length / 4, 0);
66-}
67-
68-/**
69- * Convert a DB turn to a ChatHistoryTurn for the chat function
70- */
71-function turnToHistoryTurn(turn: Turn): ChatHistoryTurn {
72- return {
73- role: turn.role,
74- content: turn.content,
75- toolCalls: turn.toolCalls as ToolCallData[] | undefined,
76- };
77-}
78-
79-/**
80- * Generate a summary of older messages using the LLM
81- */
82-async function generateSummary(
83- existingSummary: string | null,
84- turns: Turn[],
85-): Promise<string> {
86- const llm = new ChatOllama({ model: "qwen3:8b" });
87-
88- const conversationText = turns
89- .map((t) => `${t.role}: ${t.content}`)
90- .join("\n");
91-
92- const prompt = `Summarize this conversation concisely, preserving key facts, decisions, and context. Focus on information that would be useful for continuing the conversation.
93-${existingSummary ? `\nPrevious summary to incorporate:\n${existingSummary}\n` : ""}
94-Conversation to summarize:
95-${conversationText}
96-
97-Summary:`;
98-
99- const response = await llm.invoke(prompt);
100- return typeof response.content === "string"
101- ? response.content
102- : JSON.stringify(response.content);
103-}