xonecas.io
778
Pi

pi-extensions / files

pi-extensions / extensions / exa / index.ts

13.1 KB
1import { mkdtemp, writeFile } from "node:fs/promises";
2import { tmpdir } from "node:os";
3import path from "node:path";
4import { StringEnum } from "@earendil-works/pi-ai";
5import {
6	DEFAULT_MAX_BYTES,
7	DEFAULT_MAX_LINES,
8	formatSize,
9	truncateHead,
10	type ExtensionAPI,
11	withFileMutationQueue,
12} from "@earendil-works/pi-coding-agent";
13import { Type } from "typebox";
14
15const EXA_MCP_URL =
16	"https://mcp.exa.ai/mcp?tools=web_search_exa,web_fetch_exa";
17const MCP_PROTOCOL_VERSION = "2025-06-18";
18const DEFAULT_RATE_LIMIT_WAIT_MS = 1000;
19const MAX_RATE_LIMIT_WAIT_MS = 5000;
20const DEFAULT_CONTENTS_MAX_CHARACTERS = 3000;
21const SEARCH_TYPES = [
22	"auto",
23	"fast",
24	"instant",
25	"deep-lite",
26	"deep",
27	"deep-reasoning",
28] as const;
29
30const SearchParams = Type.Object({
31	query: Type.String({ description: "Natural-language web search query" }),
32	type: Type.Optional(
33		StringEnum(SEARCH_TYPES, {
34			description: "Search depth and latency tradeoff (default: auto)",
35		}),
36	),
37	numResults: Type.Optional(
38		Type.Integer({
39			minimum: 1,
40			maximum: 10,
41			description: "Number of results to return (default: 10)",
42		}),
43	),
44});
45
46const ContentsParams = Type.Object({
47	urls: Type.Array(Type.String({ description: "A URL to read" }), {
48		minItems: 1,
49		maxItems: 10,
50		description: "URLs to read (1-10). Batch multiple URLs in one call.",
51	}),
52	maxCharacters: Type.Optional(
53		Type.Integer({
54			minimum: 1,
55			description: `Maximum characters to extract per page (default: ${DEFAULT_CONTENTS_MAX_CHARACTERS})`,
56		}),
57	),
58});
59
60interface JsonRpcResponse {
61	jsonrpc: "2.0";
62	id?: number;
63	result?: unknown;
64	error?: { code: number; message: string };
65}
66
67interface McpToolResult {
68	content?: Array<{ type: string; text?: string }>;
69	isError?: boolean;
70}
71
72interface McpCallOutcome {
73	text: string;
74	keyed: boolean;
75	rateLimitRetries: number;
76}
77
78class McpError extends Error {
79	readonly status: number | undefined;
80	readonly rpcCode: number | undefined;
81	readonly retryAfterMs: number | undefined;
82
83	constructor(
84		message: string,
85		options?: {
86			status?: number;
87			rpcCode?: number;
88			retryAfterMs?: number;
89		},
90	) {
91		super(message);
92		this.name = "McpError";
93		this.status = options?.status;
94		this.rpcCode = options?.rpcCode;
95		this.retryAfterMs = options?.retryAfterMs;
96	}
97}
98
99function isRecord(value: unknown): value is Record<string, unknown> {
100	return typeof value === "object" && value !== null && !Array.isArray(value);
101}
102
103function isMcpToolResult(value: unknown): value is McpToolResult {
104	if (!isRecord(value)) return false;
105	if (value.content !== undefined) {
106		if (!Array.isArray(value.content)) return false;
107		if (
108			!value.content.every(
109				(block) =>
110					isRecord(block) &&
111					typeof block.type === "string" &&
112					(block.text === undefined || typeof block.text === "string"),
113			)
114		) {
115			return false;
116		}
117	}
118	return value.isError === undefined || typeof value.isError === "boolean";
119}
120
121function apiError(body: string): string {
122	try {
123		const parsed: unknown = JSON.parse(body);
124		if (isRecord(parsed)) {
125			if (typeof parsed.error === "string") return parsed.error;
126			if (
127				isRecord(parsed.error) &&
128				typeof parsed.error.message === "string"
129			) {
130				return parsed.error.message;
131			}
132		}
133	} catch {
134		// Fall back to the response body below.
135	}
136	return body.trim() || "Empty response";
137}
138
139function parseMcpResponse(
140	body: string,
141	contentType: string,
142	requestId: number,
143): JsonRpcResponse {
144	const candidates = contentType.includes("text/event-stream")
145		? body
146				.split("\n")
147				.filter((line) => line.startsWith("data:"))
148				.map((line) => line.slice(5).trim())
149		: [body];
150
151	for (const candidate of candidates.reverse()) {
152		if (!candidate || candidate === "[DONE]") continue;
153		try {
154			const parsed = JSON.parse(candidate) as JsonRpcResponse;
155			if (parsed.jsonrpc === "2.0" && parsed.id === requestId) return parsed;
156		} catch {
157			// Ignore non-JSON SSE data and continue looking for our response.
158		}
159	}
160	throw new Error("Exa MCP returned no matching JSON-RPC response");
161}
162
163function parseRetryAfterMs(header: string | null): number | undefined {
164	if (!header) return undefined;
165	const seconds = Number.parseFloat(header);
166	if (!Number.isFinite(seconds) || seconds < 0) return undefined;
167	return seconds * 1000;
168}
169
170function rateLimitWaitMs(error: unknown): number | undefined {
171	if (!(error instanceof McpError)) return undefined;
172	if (error.status === 429) {
173		return error.retryAfterMs ?? DEFAULT_RATE_LIMIT_WAIT_MS;
174	}
175	// The server has returned code -32000 with a rate-limit message before;
176	// catch it if it ever arrives wrapped in an HTTP 200. The message check
177	// requires an HTTP or RPC signal, so tool-level content that mentions
178	// "rate limit" does not trigger a paid retry.
179	if (
180		error.rpcCode === -32000 ||
181		((error.status !== undefined || error.rpcCode !== undefined) &&
182			/rate.?limit/i.test(error.message))
183	) {
184		return error.retryAfterMs ?? DEFAULT_RATE_LIMIT_WAIT_MS;
185	}
186	return undefined;
187}
188
189function sleep(ms: number, signal: AbortSignal | undefined): Promise<void> {
190	return new Promise((resolve, reject) => {
191		if (signal?.aborted) {
192			reject(new Error("Aborted"));
193			return;
194		}
195		const timer = setTimeout(() => {
196			signal?.removeEventListener("abort", onAbort);
197			resolve();
198		}, ms);
199		function onAbort() {
200			clearTimeout(timer);
201			reject(new Error("Aborted"));
202		}
203		signal?.addEventListener("abort", onAbort, { once: true });
204	});
205}
206
207function mcpHeaders(apiKey: string | undefined): Record<string, string> {
208	const headers: Record<string, string> = {
209		"Content-Type": "application/json",
210		Accept: "application/json, text/event-stream",
211	};
212	if (apiKey) headers["x-api-key"] = apiKey;
213	return headers;
214}
215
216async function mcpRequest(
217	endpoint: string,
218	requestId: number,
219	method: string,
220	params: Record<string, unknown>,
221	sessionId: string | undefined,
222	apiKey: string | undefined,
223	signal: AbortSignal | undefined,
224): Promise<{ response: JsonRpcResponse; sessionId?: string }> {
225	const headers = mcpHeaders(apiKey);
226	if (sessionId) headers["Mcp-Session-Id"] = sessionId;
227
228	const httpResponse = await fetch(endpoint, {
229		method: "POST",
230		headers,
231		body: JSON.stringify({
232			jsonrpc: "2.0",
233			id: requestId,
234			method,
235			params,
236		}),
237		signal,
238	});
239	const body = await httpResponse.text();
240	if (!httpResponse.ok) {
241		throw new McpError(
242			`Exa MCP failed (${httpResponse.status} ${httpResponse.statusText}): ${apiError(body)}`,
243			{
244				status: httpResponse.status,
245				retryAfterMs: parseRetryAfterMs(
246					httpResponse.headers.get("Retry-After"),
247				),
248			},
249		);
250	}
251
252	const response = parseMcpResponse(
253		body,
254		httpResponse.headers.get("Content-Type") ?? "",
255		requestId,
256	);
257	if (response.error) {
258		throw new McpError(
259			`Exa MCP error ${response.error.code}: ${response.error.message}`,
260			{
261				rpcCode: response.error.code,
262				retryAfterMs: parseRetryAfterMs(
263					httpResponse.headers.get("Retry-After"),
264				),
265			},
266		);
267	}
268	return {
269		response,
270		sessionId:
271			httpResponse.headers.get("Mcp-Session-Id") ?? sessionId ?? undefined,
272	};
273}
274
275async function notifyMcpInitialized(
276	endpoint: string,
277	sessionId: string,
278	apiKey: string | undefined,
279	signal: AbortSignal | undefined,
280): Promise<void> {
281	const headers = mcpHeaders(apiKey);
282	headers["Mcp-Session-Id"] = sessionId;
283
284	const response = await fetch(endpoint, {
285		method: "POST",
286		headers,
287		body: JSON.stringify({
288			jsonrpc: "2.0",
289			method: "notifications/initialized",
290			params: {},
291		}),
292		signal,
293	});
294	const body = await response.text();
295	if (!response.ok) {
296		throw new McpError(
297			`Exa MCP initialization failed (${response.status} ${response.statusText}): ${apiError(body)}`,
298			{
299				status: response.status,
300				retryAfterMs: parseRetryAfterMs(
301					response.headers.get("Retry-After"),
302				),
303			},
304		);
305	}
306}
307
308async function mcpToolCall(
309	toolName: string,
310	args: Record<string, unknown>,
311	urlParams: Record<string, string>,
312	apiKey: string | undefined,
313	signal: AbortSignal | undefined,
314): Promise<string> {
315	const endpoint = new URL(EXA_MCP_URL);
316	for (const [name, value] of Object.entries(urlParams)) {
317		endpoint.searchParams.set(name, value);
318	}
319
320	const initialized = await mcpRequest(
321		endpoint.toString(),
322		1,
323		"initialize",
324		{
325			protocolVersion: MCP_PROTOCOL_VERSION,
326			capabilities: {},
327			clientInfo: { name: "pi-exa-extension", version: "1.0.0" },
328		},
329		undefined,
330		apiKey,
331		signal,
332	);
333	if (!initialized.sessionId) {
334		throw new Error("Exa MCP did not return a session ID");
335	}
336	await notifyMcpInitialized(
337		endpoint.toString(),
338		initialized.sessionId,
339		apiKey,
340		signal,
341	);
342
343	const called = await mcpRequest(
344		endpoint.toString(),
345		2,
346		"tools/call",
347		{ name: toolName, arguments: args },
348		initialized.sessionId,
349		apiKey,
350		signal,
351	);
352	if (!isMcpToolResult(called.response.result)) {
353		throw new Error("Exa MCP returned an invalid tool result");
354	}
355	const result = called.response.result;
356	const text = (result.content ?? [])
357		.filter(
358			(block): block is { type: string; text: string } =>
359				block.type === "text" && typeof block.text === "string",
360		)
361		.map((block) => block.text)
362		.join("\n")
363		.trim();
364	if (result.isError) throw new McpError(text || "Exa MCP tool call failed");
365	return text || "No results found.";
366}
367
368async function callMcpTool(
369	toolName: string,
370	args: Record<string, unknown>,
371	urlParams: Record<string, string>,
372	signal: AbortSignal | undefined,
373): Promise<McpCallOutcome> {
374	const apiKey = process.env.EXA_API_KEY?.trim() || undefined;
375	if (!apiKey) {
376		const text = await mcpToolCall(
377			toolName,
378			args,
379			urlParams,
380			undefined,
381			signal,
382		);
383		return { text, keyed: false, rateLimitRetries: 0 };
384	}
385
386	// Prefer the free anonymous tier. On a rate limit, wait and retry once,
387	// then fall back to the API key. Other errors and keyed failures surface
388	// without further fallbacks.
389	let rateLimitRetries = 0;
390	for (;;) {
391		try {
392			const text = await mcpToolCall(
393				toolName,
394				args,
395				urlParams,
396				undefined,
397				signal,
398			);
399			return { text, keyed: false, rateLimitRetries };
400		} catch (error) {
401			const waitMs = rateLimitWaitMs(error);
402			if (waitMs === undefined) throw error;
403			rateLimitRetries += 1;
404			if (rateLimitRetries > 1) break;
405			await sleep(Math.min(waitMs, MAX_RATE_LIMIT_WAIT_MS), signal);
406		}
407	}
408	const text = await mcpToolCall(toolName, args, urlParams, apiKey, signal);
409	return { text, keyed: true, rateLimitRetries };
410}
411
412async function buildToolOutput(
413	output: string,
414): Promise<{ text: string; fullOutputPath?: string }> {
415	const truncation = truncateHead(output, {
416		maxLines: DEFAULT_MAX_LINES,
417		maxBytes: DEFAULT_MAX_BYTES,
418	});
419	let text = truncation.content;
420	let fullOutputPath: string | undefined;
421
422	if (truncation.truncated) {
423		const directory = await mkdtemp(path.join(tmpdir(), "pi-exa-"));
424		const tempFile = path.join(directory, "results.txt");
425		fullOutputPath = tempFile;
426		await withFileMutationQueue(tempFile, () =>
427			writeFile(tempFile, output, "utf8"),
428		);
429		text += `\n\n[Output truncated: showing ${truncation.outputLines} of ${truncation.totalLines} lines (${formatSize(truncation.outputBytes)} of ${formatSize(truncation.totalBytes)}). Full output saved to: ${fullOutputPath}]`;
430	}
431	return { text, fullOutputPath };
432}
433
434export default function exaExtension(pi: ExtensionAPI): void {
435	pi.registerTool({
436		name: "exa_search",
437		label: "Exa Search",
438		description: `Search the web with Exa and return URLs plus relevant highlights. Output is truncated to ${DEFAULT_MAX_LINES} lines or ${formatSize(DEFAULT_MAX_BYTES)}. Uses Exa's free hosted MCP service; EXA_API_KEY is optional and only used as a rate-limit fallback.`,
439		promptSnippet:
440			"Search the web with Exa and return relevant excerpts and source URLs",
441		promptGuidelines: [
442			"Use exa_search for current information and web research, and cite the returned source URLs in the answer.",
443		],
444		parameters: SearchParams,
445		async execute(_toolCallId, params, signal) {
446			const searchType = params.type ?? "auto";
447			const result = await callMcpTool(
448				"web_search_exa",
449				{ query: params.query, numResults: params.numResults ?? 10 },
450				{ defaultSearchType: searchType },
451				signal,
452			);
453			const { text, fullOutputPath } = await buildToolOutput(result.text);
454			return {
455				content: [{ type: "text", text }],
456				details: {
457					keyed: result.keyed,
458					rateLimitRetries: result.rateLimitRetries,
459					requestedSearchType: searchType,
460					fullOutputPath,
461				},
462			};
463		},
464	});
465
466	pi.registerTool({
467		name: "exa_contents",
468		label: "Exa Contents",
469		description:
470			"Read the full content of web pages as clean markdown with Exa, when you know the URLs.",
471		promptSnippet: "Read full web page content for known URLs with Exa",
472		promptGuidelines: [
473			"Use exa_contents to read the full content of pages whose URLs you know.",
474		],
475		parameters: ContentsParams,
476		async execute(_toolCallId, params, signal) {
477			const result = await callMcpTool(
478				"web_fetch_exa",
479				{
480					urls: params.urls,
481					maxCharacters:
482						params.maxCharacters ?? DEFAULT_CONTENTS_MAX_CHARACTERS,
483				},
484				{},
485				signal,
486			);
487			const { text, fullOutputPath } = await buildToolOutput(result.text);
488			return {
489				content: [{ type: "text", text }],
490				details: {
491					keyed: result.keyed,
492					rateLimitRetries: result.rateLimitRetries,
493					urlCount: params.urls.length,
494					fullOutputPath,
495				},
496			};
497		},
498	});
499}