@@ -12,9 +12,12 @@ import {
1212 } from "@earendil-works/pi-coding-agent";
1313 import { Type } from "typebox";
1414
15-const EXA_SEARCH_URL = "https://api.exa.ai/search";
16-const EXA_MCP_URL = "https://mcp.exa.ai/mcp";
15+const EXA_MCP_URL =
16+ "https://mcp.exa.ai/mcp?tools=web_search_exa,web_fetch_exa";
1717 const MCP_PROTOCOL_VERSION = "2025-06-18";
18+const DEFAULT_RATE_LIMIT_WAIT_MS = 1000;
19+const MAX_RATE_LIMIT_WAIT_MS = 5000;
20+const DEFAULT_CONTENTS_MAX_CHARACTERS = 3000;
1821 const SEARCH_TYPES = [
1922 "auto",
2023 "fast",
@@ -28,8 +31,7 @@ const SearchParams = Type.Object({
2831 query: Type.String({ description: "Natural-language web search query" }),
2932 type: Type.Optional(
3033 StringEnum(SEARCH_TYPES, {
31- description:
32- "Search depth and latency tradeoff (default: auto). Deep types require EXA_API_KEY.",
34+ description: "Search depth and latency tradeoff (default: auto)",
3335 }),
3436 ),
3537 numResults: Type.Optional(
@@ -41,12 +43,19 @@ const SearchParams = Type.Object({
4143 ),
4244 });
4345
44-interface ExaResponse {
45- requestId?: string;
46- resolvedSearchType?: string;
47- results?: unknown[];
48- costDollars?: unknown;
49-}
46+const ContentsParams = Type.Object({
47+ urls: Type.Array(Type.String({ description: "A URL to read" }), {
48+ minItems: 1,
49+ maxItems: 10,
50+ description: "URLs to read (1-10). Batch multiple URLs in one call.",
51+ }),
52+ maxCharacters: Type.Optional(
53+ Type.Integer({
54+ minimum: 1,
55+ description: `Maximum characters to extract per page (default: ${DEFAULT_CONTENTS_MAX_CHARACTERS})`,
56+ }),
57+ ),
58+});
5059
5160 interface JsonRpcResponse {
5261 jsonrpc: "2.0";
@@ -60,6 +69,33 @@ interface McpToolResult {
6069 isError?: boolean;
6170 }
6271
72+interface McpCallOutcome {
73+ text: string;
74+ keyed: boolean;
75+ rateLimitRetries: number;
76+}
77+
78+class McpError extends Error {
79+ readonly status: number | undefined;
80+ readonly rpcCode: number | undefined;
81+ readonly retryAfterMs: number | undefined;
82+
83+ constructor(
84+ message: string,
85+ options?: {
86+ status?: number;
87+ rpcCode?: number;
88+ retryAfterMs?: number;
89+ },
90+ ) {
91+ super(message);
92+ this.name = "McpError";
93+ this.status = options?.status;
94+ this.rpcCode = options?.rpcCode;
95+ this.retryAfterMs = options?.retryAfterMs;
96+ }
97+}
98+
6399 function isRecord(value: unknown): value is Record<string, unknown> {
64100 return typeof value === "object" && value !== null && !Array.isArray(value);
65101 }
@@ -124,18 +160,69 @@ function parseMcpResponse(
124160 throw new Error("Exa MCP returned no matching JSON-RPC response");
125161 }
126162
163+function parseRetryAfterMs(header: string | null): number | undefined {
164+ if (!header) return undefined;
165+ const seconds = Number.parseFloat(header);
166+ if (!Number.isFinite(seconds) || seconds < 0) return undefined;
167+ return seconds * 1000;
168+}
169+
170+function rateLimitWaitMs(error: unknown): number | undefined {
171+ if (!(error instanceof McpError)) return undefined;
172+ if (error.status === 429) {
173+ return error.retryAfterMs ?? DEFAULT_RATE_LIMIT_WAIT_MS;
174+ }
175+ // The server has returned code -32000 with a rate-limit message before;
176+ // catch it if it ever arrives wrapped in an HTTP 200. The message check
177+ // requires an HTTP or RPC signal, so tool-level content that mentions
178+ // "rate limit" does not trigger a paid retry.
179+ if (
180+ error.rpcCode === -32000 ||
181+ ((error.status !== undefined || error.rpcCode !== undefined) &&
182+ /rate.?limit/i.test(error.message))
183+ ) {
184+ return error.retryAfterMs ?? DEFAULT_RATE_LIMIT_WAIT_MS;
185+ }
186+ return undefined;
187+}
188+
189+function sleep(ms: number, signal: AbortSignal | undefined): Promise<void> {
190+ return new Promise((resolve, reject) => {
191+ if (signal?.aborted) {
192+ reject(new Error("Aborted"));
193+ return;
194+ }
195+ const timer = setTimeout(() => {
196+ signal?.removeEventListener("abort", onAbort);
197+ resolve();
198+ }, ms);
199+ function onAbort() {
200+ clearTimeout(timer);
201+ reject(new Error("Aborted"));
202+ }
203+ signal?.addEventListener("abort", onAbort, { once: true });
204+ });
205+}
206+
207+function mcpHeaders(apiKey: string | undefined): Record<string, string> {
208+ const headers: Record<string, string> = {
209+ "Content-Type": "application/json",
210+ Accept: "application/json, text/event-stream",
211+ };
212+ if (apiKey) headers["x-api-key"] = apiKey;
213+ return headers;
214+}
215+
127216 async function mcpRequest(
128217 endpoint: string,
129218 requestId: number,
130219 method: string,
131220 params: Record<string, unknown>,
132221 sessionId: string | undefined,
222+ apiKey: string | undefined,
133223 signal: AbortSignal | undefined,
134224 ): Promise<{ response: JsonRpcResponse; sessionId?: string }> {
135- const headers: Record<string, string> = {
136- "Content-Type": "application/json",
137- Accept: "application/json, text/event-stream",
138- };
225+ const headers = mcpHeaders(apiKey);
139226 if (sessionId) headers["Mcp-Session-Id"] = sessionId;
140227
141228 const httpResponse = await fetch(endpoint, {
@@ -151,8 +238,14 @@ async function mcpRequest(
151238 });
152239 const body = await httpResponse.text();
153240 if (!httpResponse.ok) {
154- throw new Error(
241+ throw new McpError(
155242 `Exa MCP failed (${httpResponse.status} ${httpResponse.statusText}): ${apiError(body)}`,
243+ {
244+ status: httpResponse.status,
245+ retryAfterMs: parseRetryAfterMs(
246+ httpResponse.headers.get("Retry-After"),
247+ ),
248+ },
156249 );
157250 }
158251
@@ -162,8 +255,14 @@ async function mcpRequest(
162255 requestId,
163256 );
164257 if (response.error) {
165- throw new Error(
258+ throw new McpError(
166259 `Exa MCP error ${response.error.code}: ${response.error.message}`,
260+ {
261+ rpcCode: response.error.code,
262+ retryAfterMs: parseRetryAfterMs(
263+ httpResponse.headers.get("Retry-After"),
264+ ),
265+ },
167266 );
168267 }
169268 return {
@@ -176,15 +275,15 @@ async function mcpRequest(
176275 async function notifyMcpInitialized(
177276 endpoint: string,
178277 sessionId: string,
278+ apiKey: string | undefined,
179279 signal: AbortSignal | undefined,
180280 ): Promise<void> {
281+ const headers = mcpHeaders(apiKey);
282+ headers["Mcp-Session-Id"] = sessionId;
283+
181284 const response = await fetch(endpoint, {
182285 method: "POST",
183- headers: {
184- "Content-Type": "application/json",
185- Accept: "application/json, text/event-stream",
186- "Mcp-Session-Id": sessionId,
187- },
286+ headers,
188287 body: JSON.stringify({
189288 jsonrpc: "2.0",
190289 method: "notifications/initialized",
@@ -192,33 +291,31 @@ async function notifyMcpInitialized(
192291 }),
193292 signal,
194293 });
294+ const body = await response.text();
195295 if (!response.ok) {
196- throw new Error(
197- `Exa MCP initialization failed (${response.status} ${response.statusText}): ${await response.text()}`,
296+ throw new McpError(
297+ `Exa MCP initialization failed (${response.status} ${response.statusText}): ${apiError(body)}`,
298+ {
299+ status: response.status,
300+ retryAfterMs: parseRetryAfterMs(
301+ response.headers.get("Retry-After"),
302+ ),
303+ },
198304 );
199305 }
200- await response.text();
201306 }
202307
203-async function searchAnonymously(
204- query: string,
205- numResults: number,
206- searchType: (typeof SEARCH_TYPES)[number],
308+async function mcpToolCall(
309+ toolName: string,
310+ args: Record<string, unknown>,
311+ urlParams: Record<string, string>,
312+ apiKey: string | undefined,
207313 signal: AbortSignal | undefined,
208314 ): Promise<string> {
209- if (
210- searchType !== "auto" &&
211- searchType !== "fast" &&
212- searchType !== "instant"
213- ) {
214- throw new Error(
215- `Anonymous Exa search does not support type "${searchType}"; set EXA_API_KEY to use deep search types`,
216- );
217- }
218-
219315 const endpoint = new URL(EXA_MCP_URL);
220- endpoint.searchParams.set("tools", "web_search_exa");
221- endpoint.searchParams.set("defaultSearchType", searchType);
316+ for (const [name, value] of Object.entries(urlParams)) {
317+ endpoint.searchParams.set(name, value);
318+ }
222319
223320 const initialized = await mcpRequest(
224321 endpoint.toString(),
@@ -230,22 +327,26 @@ async function searchAnonymously(
230327 clientInfo: { name: "pi-exa-extension", version: "1.0.0" },
231328 },
232329 undefined,
330+ apiKey,
233331 signal,
234332 );
235333 if (!initialized.sessionId) {
236334 throw new Error("Exa MCP did not return a session ID");
237335 }
238- await notifyMcpInitialized(endpoint.toString(), initialized.sessionId, signal);
336+ await notifyMcpInitialized(
337+ endpoint.toString(),
338+ initialized.sessionId,
339+ apiKey,
340+ signal,
341+ );
239342
240343 const called = await mcpRequest(
241344 endpoint.toString(),
242345 2,
243346 "tools/call",
244- {
245- name: "web_search_exa",
246- arguments: { query, numResults },
247- },
347+ { name: toolName, arguments: args },
248348 initialized.sessionId,
349+ apiKey,
249350 signal,
250351 );
251352 if (!isMcpToolResult(called.response.result)) {
@@ -260,100 +361,138 @@ async function searchAnonymously(
260361 .map((block) => block.text)
261362 .join("\n")
262363 .trim();
263- if (result.isError) throw new Error(text || "Exa MCP search failed");
364+ if (result.isError) throw new McpError(text || "Exa MCP tool call failed");
264365 return text || "No results found.";
265366 }
266367
368+async function callMcpTool(
369+ toolName: string,
370+ args: Record<string, unknown>,
371+ urlParams: Record<string, string>,
372+ signal: AbortSignal | undefined,
373+): Promise<McpCallOutcome> {
374+ const apiKey = process.env.EXA_API_KEY?.trim() || undefined;
375+ if (!apiKey) {
376+ const text = await mcpToolCall(
377+ toolName,
378+ args,
379+ urlParams,
380+ undefined,
381+ signal,
382+ );
383+ return { text, keyed: false, rateLimitRetries: 0 };
384+ }
385+
386+ // Prefer the free anonymous tier. On a rate limit, wait and retry once,
387+ // then fall back to the API key. Other errors and keyed failures surface
388+ // without further fallbacks.
389+ let rateLimitRetries = 0;
390+ for (;;) {
391+ try {
392+ const text = await mcpToolCall(
393+ toolName,
394+ args,
395+ urlParams,
396+ undefined,
397+ signal,
398+ );
399+ return { text, keyed: false, rateLimitRetries };
400+ } catch (error) {
401+ const waitMs = rateLimitWaitMs(error);
402+ if (waitMs === undefined) throw error;
403+ rateLimitRetries += 1;
404+ if (rateLimitRetries > 1) break;
405+ await sleep(Math.min(waitMs, MAX_RATE_LIMIT_WAIT_MS), signal);
406+ }
407+ }
408+ const text = await mcpToolCall(toolName, args, urlParams, apiKey, signal);
409+ return { text, keyed: true, rateLimitRetries };
410+}
411+
412+async function buildToolOutput(
413+ output: string,
414+): Promise<{ text: string; fullOutputPath?: string }> {
415+ const truncation = truncateHead(output, {
416+ maxLines: DEFAULT_MAX_LINES,
417+ maxBytes: DEFAULT_MAX_BYTES,
418+ });
419+ let text = truncation.content;
420+ let fullOutputPath: string | undefined;
421+
422+ if (truncation.truncated) {
423+ const directory = await mkdtemp(path.join(tmpdir(), "pi-exa-"));
424+ const tempFile = path.join(directory, "results.txt");
425+ fullOutputPath = tempFile;
426+ await withFileMutationQueue(tempFile, () =>
427+ writeFile(tempFile, output, "utf8"),
428+ );
429+ text += `\n\n[Output truncated: showing ${truncation.outputLines} of ${truncation.totalLines} lines (${formatSize(truncation.outputBytes)} of ${formatSize(truncation.totalBytes)}). Full output saved to: ${fullOutputPath}]`;
430+ }
431+ return { text, fullOutputPath };
432+}
433+
267434 export default function exaExtension(pi: ExtensionAPI): void {
268435 pi.registerTool({
269436 name: "exa_search",
270437 label: "Exa Search",
271- description: `Search the web with Exa and return URLs plus relevant highlights. Output is truncated to ${DEFAULT_MAX_LINES} lines or ${formatSize(DEFAULT_MAX_BYTES)}. Uses the Exa API with EXA_API_KEY, or Exa's free hosted MCP service anonymously.`,
272- promptSnippet: "Search the web with Exa and return relevant excerpts and source URLs",
438+ description: `Search the web with Exa and return URLs plus relevant highlights. Output is truncated to ${DEFAULT_MAX_LINES} lines or ${formatSize(DEFAULT_MAX_BYTES)}. Uses Exa's free hosted MCP service; EXA_API_KEY is optional and only used as a rate-limit fallback.`,
439+ promptSnippet:
440+ "Search the web with Exa and return relevant excerpts and source URLs",
273441 promptGuidelines: [
274442 "Use exa_search for current information and web research, and cite the returned source URLs in the answer.",
275443 ],
276444 parameters: SearchParams,
277445 async execute(_toolCallId, params, signal) {
278- const apiKey = process.env.EXA_API_KEY?.trim();
279446 const searchType = params.type ?? "auto";
280- const numResults = params.numResults ?? 10;
281- let output: string;
282- let details: Record<string, unknown>;
283-
284- if (apiKey) {
285- const response = await fetch(EXA_SEARCH_URL, {
286- method: "POST",
287- headers: {
288- "Content-Type": "application/json",
289- Authorization: `Bearer ${apiKey}`,
290- },
291- body: JSON.stringify({
292- query: params.query,
293- type: searchType,
294- numResults,
295- contents: { highlights: true },
296- }),
297- signal,
298- });
299- const body = await response.text();
300- if (!response.ok) {
301- throw new Error(
302- `Exa search failed (${response.status} ${response.statusText}): ${apiError(body)}`,
303- );
304- }
305-
306- let data: ExaResponse;
307- try {
308- data = JSON.parse(body) as ExaResponse;
309- } catch {
310- throw new Error("Exa returned an invalid JSON response");
311- }
312- if (!Array.isArray(data.results)) {
313- throw new Error("Exa response did not include a results array");
314- }
315- output = JSON.stringify(data, null, 2);
316- details = {
317- backend: "api",
318- requestId: data.requestId,
319- requestedSearchType: searchType,
320- resolvedSearchType: data.resolvedSearchType,
321- resultCount: data.results.length,
322- costDollars: data.costDollars,
323- };
324- } else {
325- output = await searchAnonymously(
326- params.query,
327- numResults,
328- searchType,
329- signal,
330- );
331- details = {
332- backend: "mcp-free",
447+ const result = await callMcpTool(
448+ "web_search_exa",
449+ { query: params.query, numResults: params.numResults ?? 10 },
450+ { defaultSearchType: searchType },
451+ signal,
452+ );
453+ const { text, fullOutputPath } = await buildToolOutput(result.text);
454+ return {
455+ content: [{ type: "text", text }],
456+ details: {
457+ keyed: result.keyed,
458+ rateLimitRetries: result.rateLimitRetries,
333459 requestedSearchType: searchType,
334- };
335- }
336-
337- const truncation = truncateHead(output, {
338- maxLines: DEFAULT_MAX_LINES,
339- maxBytes: DEFAULT_MAX_BYTES,
340- });
341- let text = truncation.content;
342- let fullOutputPath: string | undefined;
343-
344- if (truncation.truncated) {
345- const directory = await mkdtemp(path.join(tmpdir(), "pi-exa-"));
346- const tempFile = path.join(directory, "results.txt");
347- fullOutputPath = tempFile;
348- await withFileMutationQueue(tempFile, () =>
349- writeFile(tempFile, output, "utf8"),
350- );
351- text += `\n\n[Output truncated: showing ${truncation.outputLines} of ${truncation.totalLines} lines (${formatSize(truncation.outputBytes)} of ${formatSize(truncation.totalBytes)}). Full output saved to: ${fullOutputPath}]`;
352- }
460+ fullOutputPath,
461+ },
462+ };
463+ },
464+ });
353465
466+ pi.registerTool({
467+ name: "exa_contents",
468+ label: "Exa Contents",
469+ description:
470+ "Read the full content of web pages as clean markdown with Exa, when you know the URLs.",
471+ promptSnippet: "Read full web page content for known URLs with Exa",
472+ promptGuidelines: [
473+ "Use exa_contents to read the full content of pages whose URLs you know.",
474+ ],
475+ parameters: ContentsParams,
476+ async execute(_toolCallId, params, signal) {
477+ const result = await callMcpTool(
478+ "web_fetch_exa",
479+ {
480+ urls: params.urls,
481+ maxCharacters:
482+ params.maxCharacters ?? DEFAULT_CONTENTS_MAX_CHARACTERS,
483+ },
484+ {},
485+ signal,
486+ );
487+ const { text, fullOutputPath } = await buildToolOutput(result.text);
354488 return {
355489 content: [{ type: "text", text }],
356- details: { ...details, fullOutputPath },
490+ details: {
491+ keyed: result.keyed,
492+ rateLimitRetries: result.rateLimitRetries,
493+ urlCount: params.urls.length,
494+ fullOutputPath,
495+ },
357496 };
358497 },
359498 });