778
Pi
pi-extensions / files
pi-extensions / extensions / exa / index.ts
13.1 KB
1import { mkdtemp, writeFile } from "node:fs/promises"; 2import { tmpdir } from "node:os"; 3import path from "node:path"; 4import { StringEnum } from "@earendil-works/pi-ai"; 5import { 6 DEFAULT_MAX_BYTES, 7 DEFAULT_MAX_LINES, 8 formatSize, 9 truncateHead, 10 type ExtensionAPI, 11 withFileMutationQueue, 12} from "@earendil-works/pi-coding-agent"; 13import { Type } from "typebox"; 14 15const EXA_MCP_URL = 16 "https://mcp.exa.ai/mcp?tools=web_search_exa,web_fetch_exa"; 17const MCP_PROTOCOL_VERSION = "2025-06-18"; 18const DEFAULT_RATE_LIMIT_WAIT_MS = 1000; 19const MAX_RATE_LIMIT_WAIT_MS = 5000; 20const DEFAULT_CONTENTS_MAX_CHARACTERS = 3000; 21const SEARCH_TYPES = [ 22 "auto", 23 "fast", 24 "instant", 25 "deep-lite", 26 "deep", 27 "deep-reasoning", 28] as const; 29 30const SearchParams = Type.Object({ 31 query: Type.String({ description: "Natural-language web search query" }), 32 type: Type.Optional( 33 StringEnum(SEARCH_TYPES, { 34 description: "Search depth and latency tradeoff (default: auto)", 35 }), 36 ), 37 numResults: Type.Optional( 38 Type.Integer({ 39 minimum: 1, 40 maximum: 10, 41 description: "Number of results to return (default: 10)", 42 }), 43 ), 44}); 45 46const ContentsParams = Type.Object({ 47 urls: Type.Array(Type.String({ description: "A URL to read" }), { 48 minItems: 1, 49 maxItems: 10, 50 description: "URLs to read (1-10). Batch multiple URLs in one call.", 51 }), 52 maxCharacters: Type.Optional( 53 Type.Integer({ 54 minimum: 1, 55 description: `Maximum characters to extract per page (default: ${DEFAULT_CONTENTS_MAX_CHARACTERS})`, 56 }), 57 ), 58}); 59 60interface JsonRpcResponse { 61 jsonrpc: "2.0"; 62 id?: number; 63 result?: unknown; 64 error?: { code: number; message: string }; 65} 66 67interface McpToolResult { 68 content?: Array<{ type: string; text?: string }>; 69 isError?: boolean; 70} 71 72interface McpCallOutcome { 73 text: string; 74 keyed: boolean; 75 rateLimitRetries: number; 76} 77 78class McpError extends Error { 79 readonly status: number | undefined; 80 readonly rpcCode: number | undefined; 81 readonly retryAfterMs: number | undefined; 82 83 constructor( 84 message: string, 85 options?: { 86 status?: number; 87 rpcCode?: number; 88 retryAfterMs?: number; 89 }, 90 ) { 91 super(message); 92 this.name = "McpError"; 93 this.status = options?.status; 94 this.rpcCode = options?.rpcCode; 95 this.retryAfterMs = options?.retryAfterMs; 96 } 97} 98 99function isRecord(value: unknown): value is Record<string, unknown> { 100 return typeof value === "object" && value !== null && !Array.isArray(value); 101} 102 103function isMcpToolResult(value: unknown): value is McpToolResult { 104 if (!isRecord(value)) return false; 105 if (value.content !== undefined) { 106 if (!Array.isArray(value.content)) return false; 107 if ( 108 !value.content.every( 109 (block) => 110 isRecord(block) && 111 typeof block.type === "string" && 112 (block.text === undefined || typeof block.text === "string"), 113 ) 114 ) { 115 return false; 116 } 117 } 118 return value.isError === undefined || typeof value.isError === "boolean"; 119} 120 121function apiError(body: string): string { 122 try { 123 const parsed: unknown = JSON.parse(body); 124 if (isRecord(parsed)) { 125 if (typeof parsed.error === "string") return parsed.error; 126 if ( 127 isRecord(parsed.error) && 128 typeof parsed.error.message === "string" 129 ) { 130 return parsed.error.message; 131 } 132 } 133 } catch { 134 // Fall back to the response body below. 135 } 136 return body.trim() || "Empty response"; 137} 138 139function parseMcpResponse( 140 body: string, 141 contentType: string, 142 requestId: number, 143): JsonRpcResponse { 144 const candidates = contentType.includes("text/event-stream") 145 ? body 146 .split("\n") 147 .filter((line) => line.startsWith("data:")) 148 .map((line) => line.slice(5).trim()) 149 : [body]; 150 151 for (const candidate of candidates.reverse()) { 152 if (!candidate || candidate === "[DONE]") continue; 153 try { 154 const parsed = JSON.parse(candidate) as JsonRpcResponse; 155 if (parsed.jsonrpc === "2.0" && parsed.id === requestId) return parsed; 156 } catch { 157 // Ignore non-JSON SSE data and continue looking for our response. 158 } 159 } 160 throw new Error("Exa MCP returned no matching JSON-RPC response"); 161} 162 163function parseRetryAfterMs(header: string | null): number | undefined { 164 if (!header) return undefined; 165 const seconds = Number.parseFloat(header); 166 if (!Number.isFinite(seconds) || seconds < 0) return undefined; 167 return seconds * 1000; 168} 169 170function rateLimitWaitMs(error: unknown): number | undefined { 171 if (!(error instanceof McpError)) return undefined; 172 if (error.status === 429) { 173 return error.retryAfterMs ?? DEFAULT_RATE_LIMIT_WAIT_MS; 174 } 175 // The server has returned code -32000 with a rate-limit message before; 176 // catch it if it ever arrives wrapped in an HTTP 200. The message check 177 // requires an HTTP or RPC signal, so tool-level content that mentions 178 // "rate limit" does not trigger a paid retry. 179 if ( 180 error.rpcCode === -32000 || 181 ((error.status !== undefined || error.rpcCode !== undefined) && 182 /rate.?limit/i.test(error.message)) 183 ) { 184 return error.retryAfterMs ?? DEFAULT_RATE_LIMIT_WAIT_MS; 185 } 186 return undefined; 187} 188 189function sleep(ms: number, signal: AbortSignal | undefined): Promise<void> { 190 return new Promise((resolve, reject) => { 191 if (signal?.aborted) { 192 reject(new Error("Aborted")); 193 return; 194 } 195 const timer = setTimeout(() => { 196 signal?.removeEventListener("abort", onAbort); 197 resolve(); 198 }, ms); 199 function onAbort() { 200 clearTimeout(timer); 201 reject(new Error("Aborted")); 202 } 203 signal?.addEventListener("abort", onAbort, { once: true }); 204 }); 205} 206 207function mcpHeaders(apiKey: string | undefined): Record<string, string> { 208 const headers: Record<string, string> = { 209 "Content-Type": "application/json", 210 Accept: "application/json, text/event-stream", 211 }; 212 if (apiKey) headers["x-api-key"] = apiKey; 213 return headers; 214} 215 216async function mcpRequest( 217 endpoint: string, 218 requestId: number, 219 method: string, 220 params: Record<string, unknown>, 221 sessionId: string | undefined, 222 apiKey: string | undefined, 223 signal: AbortSignal | undefined, 224): Promise<{ response: JsonRpcResponse; sessionId?: string }> { 225 const headers = mcpHeaders(apiKey); 226 if (sessionId) headers["Mcp-Session-Id"] = sessionId; 227 228 const httpResponse = await fetch(endpoint, { 229 method: "POST", 230 headers, 231 body: JSON.stringify({ 232 jsonrpc: "2.0", 233 id: requestId, 234 method, 235 params, 236 }), 237 signal, 238 }); 239 const body = await httpResponse.text(); 240 if (!httpResponse.ok) { 241 throw new McpError( 242 `Exa MCP failed (${httpResponse.status} ${httpResponse.statusText}): ${apiError(body)}`, 243 { 244 status: httpResponse.status, 245 retryAfterMs: parseRetryAfterMs( 246 httpResponse.headers.get("Retry-After"), 247 ), 248 }, 249 ); 250 } 251 252 const response = parseMcpResponse( 253 body, 254 httpResponse.headers.get("Content-Type") ?? "", 255 requestId, 256 ); 257 if (response.error) { 258 throw new McpError( 259 `Exa MCP error ${response.error.code}: ${response.error.message}`, 260 { 261 rpcCode: response.error.code, 262 retryAfterMs: parseRetryAfterMs( 263 httpResponse.headers.get("Retry-After"), 264 ), 265 }, 266 ); 267 } 268 return { 269 response, 270 sessionId: 271 httpResponse.headers.get("Mcp-Session-Id") ?? sessionId ?? undefined, 272 }; 273} 274 275async function notifyMcpInitialized( 276 endpoint: string, 277 sessionId: string, 278 apiKey: string | undefined, 279 signal: AbortSignal | undefined, 280): Promise<void> { 281 const headers = mcpHeaders(apiKey); 282 headers["Mcp-Session-Id"] = sessionId; 283 284 const response = await fetch(endpoint, { 285 method: "POST", 286 headers, 287 body: JSON.stringify({ 288 jsonrpc: "2.0", 289 method: "notifications/initialized", 290 params: {}, 291 }), 292 signal, 293 }); 294 const body = await response.text(); 295 if (!response.ok) { 296 throw new McpError( 297 `Exa MCP initialization failed (${response.status} ${response.statusText}): ${apiError(body)}`, 298 { 299 status: response.status, 300 retryAfterMs: parseRetryAfterMs( 301 response.headers.get("Retry-After"), 302 ), 303 }, 304 ); 305 } 306} 307 308async function mcpToolCall( 309 toolName: string, 310 args: Record<string, unknown>, 311 urlParams: Record<string, string>, 312 apiKey: string | undefined, 313 signal: AbortSignal | undefined, 314): Promise<string> { 315 const endpoint = new URL(EXA_MCP_URL); 316 for (const [name, value] of Object.entries(urlParams)) { 317 endpoint.searchParams.set(name, value); 318 } 319 320 const initialized = await mcpRequest( 321 endpoint.toString(), 322 1, 323 "initialize", 324 { 325 protocolVersion: MCP_PROTOCOL_VERSION, 326 capabilities: {}, 327 clientInfo: { name: "pi-exa-extension", version: "1.0.0" }, 328 }, 329 undefined, 330 apiKey, 331 signal, 332 ); 333 if (!initialized.sessionId) { 334 throw new Error("Exa MCP did not return a session ID"); 335 } 336 await notifyMcpInitialized( 337 endpoint.toString(), 338 initialized.sessionId, 339 apiKey, 340 signal, 341 ); 342 343 const called = await mcpRequest( 344 endpoint.toString(), 345 2, 346 "tools/call", 347 { name: toolName, arguments: args }, 348 initialized.sessionId, 349 apiKey, 350 signal, 351 ); 352 if (!isMcpToolResult(called.response.result)) { 353 throw new Error("Exa MCP returned an invalid tool result"); 354 } 355 const result = called.response.result; 356 const text = (result.content ?? []) 357 .filter( 358 (block): block is { type: string; text: string } => 359 block.type === "text" && typeof block.text === "string", 360 ) 361 .map((block) => block.text) 362 .join("\n") 363 .trim(); 364 if (result.isError) throw new McpError(text || "Exa MCP tool call failed"); 365 return text || "No results found."; 366} 367 368async function callMcpTool( 369 toolName: string, 370 args: Record<string, unknown>, 371 urlParams: Record<string, string>, 372 signal: AbortSignal | undefined, 373): Promise<McpCallOutcome> { 374 const apiKey = process.env.EXA_API_KEY?.trim() || undefined; 375 if (!apiKey) { 376 const text = await mcpToolCall( 377 toolName, 378 args, 379 urlParams, 380 undefined, 381 signal, 382 ); 383 return { text, keyed: false, rateLimitRetries: 0 }; 384 } 385 386 // Prefer the free anonymous tier. On a rate limit, wait and retry once, 387 // then fall back to the API key. Other errors and keyed failures surface 388 // without further fallbacks. 389 let rateLimitRetries = 0; 390 for (;;) { 391 try { 392 const text = await mcpToolCall( 393 toolName, 394 args, 395 urlParams, 396 undefined, 397 signal, 398 ); 399 return { text, keyed: false, rateLimitRetries }; 400 } catch (error) { 401 const waitMs = rateLimitWaitMs(error); 402 if (waitMs === undefined) throw error; 403 rateLimitRetries += 1; 404 if (rateLimitRetries > 1) break; 405 await sleep(Math.min(waitMs, MAX_RATE_LIMIT_WAIT_MS), signal); 406 } 407 } 408 const text = await mcpToolCall(toolName, args, urlParams, apiKey, signal); 409 return { text, keyed: true, rateLimitRetries }; 410} 411 412async function buildToolOutput( 413 output: string, 414): Promise<{ text: string; fullOutputPath?: string }> { 415 const truncation = truncateHead(output, { 416 maxLines: DEFAULT_MAX_LINES, 417 maxBytes: DEFAULT_MAX_BYTES, 418 }); 419 let text = truncation.content; 420 let fullOutputPath: string | undefined; 421 422 if (truncation.truncated) { 423 const directory = await mkdtemp(path.join(tmpdir(), "pi-exa-")); 424 const tempFile = path.join(directory, "results.txt"); 425 fullOutputPath = tempFile; 426 await withFileMutationQueue(tempFile, () => 427 writeFile(tempFile, output, "utf8"), 428 ); 429 text += `\n\n[Output truncated: showing ${truncation.outputLines} of ${truncation.totalLines} lines (${formatSize(truncation.outputBytes)} of ${formatSize(truncation.totalBytes)}). Full output saved to: ${fullOutputPath}]`; 430 } 431 return { text, fullOutputPath }; 432} 433 434export default function exaExtension(pi: ExtensionAPI): void { 435 pi.registerTool({ 436 name: "exa_search", 437 label: "Exa Search", 438 description: `Search the web with Exa and return URLs plus relevant highlights. Output is truncated to ${DEFAULT_MAX_LINES} lines or ${formatSize(DEFAULT_MAX_BYTES)}. Uses Exa's free hosted MCP service; EXA_API_KEY is optional and only used as a rate-limit fallback.`, 439 promptSnippet: 440 "Search the web with Exa and return relevant excerpts and source URLs", 441 promptGuidelines: [ 442 "Use exa_search for current information and web research, and cite the returned source URLs in the answer.", 443 ], 444 parameters: SearchParams, 445 async execute(_toolCallId, params, signal) { 446 const searchType = params.type ?? "auto"; 447 const result = await callMcpTool( 448 "web_search_exa", 449 { query: params.query, numResults: params.numResults ?? 10 }, 450 { defaultSearchType: searchType }, 451 signal, 452 ); 453 const { text, fullOutputPath } = await buildToolOutput(result.text); 454 return { 455 content: [{ type: "text", text }], 456 details: { 457 keyed: result.keyed, 458 rateLimitRetries: result.rateLimitRetries, 459 requestedSearchType: searchType, 460 fullOutputPath, 461 }, 462 }; 463 }, 464 }); 465 466 pi.registerTool({ 467 name: "exa_contents", 468 label: "Exa Contents", 469 description: 470 "Read the full content of web pages as clean markdown with Exa, when you know the URLs.", 471 promptSnippet: "Read full web page content for known URLs with Exa", 472 promptGuidelines: [ 473 "Use exa_contents to read the full content of pages whose URLs you know.", 474 ], 475 parameters: ContentsParams, 476 async execute(_toolCallId, params, signal) { 477 const result = await callMcpTool( 478 "web_fetch_exa", 479 { 480 urls: params.urls, 481 maxCharacters: 482 params.maxCharacters ?? DEFAULT_CONTENTS_MAX_CHARACTERS, 483 }, 484 {}, 485 signal, 486 ); 487 const { text, fullOutputPath } = await buildToolOutput(result.text); 488 return { 489 content: [{ type: "text", text }], 490 details: { 491 keyed: result.keyed, 492 rateLimitRetries: result.rateLimitRetries, 493 urlCount: params.urls.length, 494 fullOutputPath, 495 }, 496 }; 497 }, 498 }); 499}