xonecas.io
62159
Sy

symb / commit 56193e3

feat: webfetch and exa_ai search

xonecas2026-02-13 56193e3871cf2fcc6be750aa6e389f574098ae7e parent 9b66295

11 files changed

This diff is too big to show in full. The first 1000 lines are below, and every file is listed with its counts.

cmd/symb/main.go+12−0
@@ -105,6 +105,18 @@ func main() {
105105 editHandler := mcp_tools.NewEditHandler(fileTracker)
106106 proxy.RegisterTool(editTool, editHandler.Handle)
107107
108+ // Register web tools (shared cache for fetch + search)
109+ webCache := mcp_tools.NewWebCache()
110+
111+ webFetchTool := mcp_tools.NewWebFetchTool()
112+ webFetchHandler := mcp_tools.MakeWebFetchHandler(webCache)
113+ proxy.RegisterTool(webFetchTool, webFetchHandler)
114+
115+ exaKey := creds.GetAPIKey("exa_ai")
116+ webSearchTool := mcp_tools.NewWebSearchTool()
117+ webSearchHandler := mcp_tools.MakeWebSearchHandler(webCache, exaKey, "")
118+ proxy.RegisterTool(webSearchTool, webSearchHandler)
119+
108120 // List all tools (local + upstream)
109121 tools, err := proxy.ListTools(context.Background())
110122 if err != nil {
exa_aiandwebsearch.md+62−0
@@ -0,0 +1,62 @@
1+# Web/Search Tools Feature
2+
3+## Current Architecture
4+
5+- **Credentials**: `~/.config/symb/credentials.json` with `providers.<name>.api_key` structure. Already supports arbitrary provider names — `exa_ai` fits naturally.
6+- **Tools**: Defined as `mcp.Tool` (name + description + JSON schema), registered on the `mcp.Proxy` via `RegisterTool(tool, handler)`. Handlers are `func(ctx, json.RawMessage) (*ToolResult, error)`.
7+- **Prompts**: 4 model-specific markdown files (`anthropic.md`, `gemini.md`, `qwen.md`, `gpt.md`), embedded via `//go:embed`. Each documents the available tools.
8+- **Registration**: All in `cmd/symb/main.go` — create tool definition + handler, register on proxy.
9+
10+## Exa AI API
11+
12+- **Auth**: `x-api-key: <key>` header
13+- **Key endpoint**: `POST https://api.exa.ai/search` — supports `type: "auto"`, inline `contents` extraction (text, highlights, summary), domain filtering, date filtering
14+- **Response**: Returns `results[]` with `url`, `title`, `text`, `publishedDate`, etc.
15+- Free tier: $10 credits, ~2000 searches
16+
17+## Design: Two New Tools
18+
19+### 1. `WebFetch` — HTTP fetch with HTML cleaning
20+
21+- **Purpose**: Fetch a URL and return cleaned text content (strip HTML tags, scripts, styles — preserve readable text)
22+- **Args**: `url` (required), `max_chars` (optional, default 10000)
23+- **Implementation**: Go's `net/http` GET, then HTML-to-text extraction (use `golang.org/x/net/html` tokenizer to strip tags)
24+- **Returns**: Cleaned text content, truncated to `max_chars`
25+
26+### 2. `WebSearch` — Exa AI search
27+
28+- **Purpose**: Search the web for information, documentation, etc.
29+- **Args**: `query` (required), `num_results` (optional, default 5), `type` (optional: "auto"/"fast"/"deep"), `include_domains` (optional), `include_text` (optional)
30+- **Implementation**: POST to `https://api.exa.ai/search` with `contents.text.maxCharacters` to keep tokens reasonable
31+- **Returns**: Formatted results (title, URL, snippet/text)
32+- **Requires**: Exa API key from `credentials.json` under `providers.exa_ai.api_key`
33+
34+### New file: `internal/mcp_tools/web.go`
35+
36+Both tools in one file. The Exa client is simple enough to not warrant a separate package — just HTTP POST with JSON.
37+
38+### Search cache
39+
40+In-memory cache shared by both `WebSearch` and `WebFetch`. Keyed by normalized string (lowercased + trimmed query for search, raw URL for fetch). Each entry stores the result and a timestamp. Entries older than 24 hours are considered stale and re-fetched. Simple `map` + `sync.RWMutex`, no persistence — cache resets on restart, which is fine since sessions are short-lived.
41+
42+### Credential plumbing
43+
44+`main.go` already loads credentials. Pass `creds.GetAPIKey("exa_ai")` to the WebSearch handler constructor. WebFetch needs no credentials.
45+
46+### Prompt updates
47+
48+Add `WebFetch` and `WebSearch` tool docs to all 4 prompt files. Add guidance like "Search before assuming when asked about external APIs, libraries, or current information."
49+
50+## Implementation Plan
51+
52+| Step | File(s) | Description |
53+|------|---------|-------------|
54+| 1 | `internal/mcp_tools/web.go` | `WebFetch` tool + handler (HTTP GET, HTML cleaning) |
55+| 2 | `internal/mcp_tools/web.go` | `WebSearch` tool + handler (Exa API client) |
56+| 3 | `internal/mcp_tools/web_test.go` | Tests for both tools |
57+| 4 | `cmd/symb/main.go` | Register both tools, pass Exa API key |
58+| 5 | `internal/llm/*.md` | Add tool docs + "search before assuming" guidance to all 4 prompts |
59+| 6 | `go.mod` | Add `golang.org/x/net` if needed for HTML parsing |
60+| 7 | `make lint && make test` | Verify |
61+
62+TODO: sqlite store?
go.mod+5−4
@@ -11,10 +11,9 @@ require (
1111 github.com/charmbracelet/lipgloss v1.1.0
1212 github.com/charmbracelet/x/ansi v0.11.6
1313 github.com/charmbracelet/x/exp/golden v0.0.0-20260209194814-eeb2896ac759
14- github.com/mattn/go-runewidth v0.0.19
15- github.com/rivo/uniseg v0.4.7
1614 github.com/rs/zerolog v1.34.0
1715 github.com/sashabaranov/go-openai v1.41.2
16+ golang.org/x/net v0.50.0
1817 )
1918
2019 require (
@@ -32,10 +31,12 @@ require (
3231 github.com/mattn/go-colorable v0.1.13 // indirect
3332 github.com/mattn/go-isatty v0.0.20 // indirect
3433 github.com/mattn/go-localereader v0.0.1 // indirect
34+ github.com/mattn/go-runewidth v0.0.19 // indirect
3535 github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
3636 github.com/muesli/cancelreader v0.2.2 // indirect
3737 github.com/muesli/termenv v0.16.0 // indirect
38+ github.com/rivo/uniseg v0.4.7 // indirect
3839 github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
39- golang.org/x/sys v0.38.0 // indirect
40- golang.org/x/text v0.3.8 // indirect
40+ golang.org/x/sys v0.41.0 // indirect
41+ golang.org/x/text v0.34.0 // indirect
4142 )
go.sum+6−4
@@ -72,11 +72,13 @@ github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavM
7272 github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
7373 golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
7474 golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
75+golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60=
76+golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM=
7577 golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
7678 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
7779 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
7880 golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
79-golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
80-golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
81-golang.org/x/text v0.3.8 h1:nAL+RVCQ9uMn3vJZbV+MRnydTJFPf8qqY42YiA6MrqY=
82-golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
81+golang.org/x/sys v0.41.0 h1:Ivj+2Cp/ylzLiEU89QhWblYnOE9zerudt9Ftecq2C6k=
82+golang.org/x/sys v0.41.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
83+golang.org/x/text v0.34.0 h1:oL/Qq0Kdaqxa1KbNeMKwQq0reLCCaFtqu2eNuSeNHbk=
84+golang.org/x/text v0.34.0/go.mod h1:homfLqTYRFyVYemLBFl5GgL/DWEiH5wcsQ5gSh1yziA=
internal/llm/anthropic.md+15−0
@@ -56,6 +56,21 @@ The 2-char hex hash is a content fingerprint for that line. You need both the li
5656 - Case sensitivity: `{"pattern": "Error", "case_sensitive": true}`
5757 - Limit results: `{"max_results": 50}` (default: 100)
5858
59+### `WebFetch` — Fetch a URL as clean text
60+Fetches a web page and returns its content with HTML stripped (scripts, styles removed). Results are cached for 24 hours.
61+
62+- `{"url": "https://example.com/docs"}` — fetch with default 10k char limit
63+- `{"url": "https://example.com/docs", "max_chars": 5000}` — custom limit
64+
65+### `WebSearch` — Search the web (Exa AI)
66+Search the web for documentation, APIs, libraries, or current information. Results are cached for 24 hours.
67+
68+- `{"query": "Go context package best practices"}`
69+- `{"query": "React hooks", "num_results": 3, "type": "fast"}`
70+- `{"query": "kubernetes API", "include_domains": ["kubernetes.io"]}`
71+
72+**Search before assuming** — when asked about external libraries, APIs, or current information, use WebSearch to verify rather than relying on potentially outdated knowledge.
73+
5974 ### `Edit` — Modify files using hash anchors
6075 **Prerequisite: Open the file first.** The hashes from Open output are your edit anchors.
6176
internal/llm/gemini.md+14−0
@@ -74,6 +74,20 @@ The 2-char hex hash is a content fingerprint. You need both line number and hash
7474 ```
7575 Respects `.gitignore`. Filename search (default) or content search (`content_search: true`).
7676
77+### `WebFetch` — Fetch a URL as clean text
78+Fetches a web page and returns its content with HTML stripped (scripts, styles removed). Results are cached for 24 hours.
79+```json
80+{"url": "https://example.com/docs", "max_chars": 10000}
81+```
82+
83+### `WebSearch` — Search the web (Exa AI)
84+Search the web for documentation, APIs, libraries, or current information. Results are cached for 24 hours.
85+```json
86+{"query": "search terms", "num_results": 5, "type": "auto", "include_domains": ["docs.example.com"]}
87+```
88+
89+**Search before assuming** — when asked about external libraries, APIs, or current information, use WebSearch to verify rather than relying on potentially outdated knowledge.
90+
7791 ### `Edit` — Modify files using hash anchors
7892 **Prerequisite: Open the file first.** The hashes from Open output are your edit anchors.
7993
internal/llm/gpt.md+18−0
@@ -70,6 +70,24 @@ The 2-char hex hash is a content fingerprint. You need both line number and hash
7070 ```
7171 Respects `.gitignore`. Filename search (default) or content search (`content_search: true`).
7272
73+### `WebFetch` — Fetch a URL as clean text
74+Fetches a web page and returns content with HTML stripped (scripts, styles removed). Results cached for 24 hours.
75+
76+**Parameters:**
77+```json
78+{"url": "https://example.com/docs", "max_chars": 10000}
79+```
80+
81+### `WebSearch` — Search the web (Exa AI)
82+Search the web for documentation, APIs, libraries, or current information. Results cached for 24 hours.
83+
84+**Parameters:**
85+```json
86+{"query": "search terms", "num_results": 5, "type": "auto", "include_domains": ["docs.example.com"]}
87+```
88+
89+**Search before assuming** — when asked about external libraries, APIs, or current information, use WebSearch to verify rather than relying on potentially outdated knowledge.
90+
7391 ### `Edit` — Modify files using hash anchors
7492 **Prerequisite: Open the file first.** The hashes from Open output are your edit anchors.
7593
internal/llm/qwen.md+14−0
@@ -52,6 +52,20 @@ The 2-char hex hash is a content fingerprint. You need both line number and hash
5252 ```
5353 Finds files or content. Respects `.gitignore`.
5454
55+### `WebFetch` — Fetch a URL as clean text
56+Fetches a URL, strips HTML. Cached 24h.
57+```json
58+{"url": "https://example.com/docs", "max_chars": 10000}
59+```
60+
61+### `WebSearch` — Search the web (Exa AI)
62+Search for docs, APIs, libraries, current info. Cached 24h.
63+```json
64+{"query": "search terms", "num_results": 5, "type": "auto", "include_domains": ["docs.example.com"]}
65+```
66+
67+**Search before assuming** — use WebSearch to verify external APIs/libraries rather than guessing.
68+
5569 ### `Edit` — Modify files using hash anchors
5670 **Open the file first.** One operation per call. Returns fresh hashes after each edit.
5771
internal/mcp_tools/edit.go+13−20
@@ -169,11 +169,11 @@ func (h *EditHandler) SetProgram(program *tea.Program) {
169169 func (h *EditHandler) Handle(ctx context.Context, arguments json.RawMessage) (*mcp.ToolResult, error) {
170170 var args EditArgs
171171 if err := json.Unmarshal(arguments, &args); err != nil {
172- return errResult("Invalid arguments: %v", err), nil
172+ return toolError("Invalid arguments: %v", err), nil
173173 }
174174
175175 if args.File == "" {
176- return errResult("File path cannot be empty"), nil
176+ return toolError("File path cannot be empty"), nil
177177 }
178178
179179 // Count operations — exactly one must be set
@@ -191,23 +191,23 @@ func (h *EditHandler) Handle(ctx context.Context, arguments json.RawMessage) (*m
191191 ops++
192192 }
193193 if ops != 1 {
194- return errResult("Exactly one operation (replace, insert, delete, or create) must be specified"), nil
194+ return toolError("Exactly one operation (replace, insert, delete, or create) must be specified"), nil
195195 }
196196
197197 // Security: validate path
198198 absPath, err := filepath.Abs(args.File)
199199 if err != nil {
200- return errResult("Invalid file path: %v", err), nil
200+ return toolError("Invalid file path: %v", err), nil
201201 }
202202
203203 workingDir, err := os.Getwd()
204204 if err != nil {
205- return errResult("Failed to get working directory: %v", err), nil
205+ return toolError("Failed to get working directory: %v", err), nil
206206 }
207207
208208 relPath, err := filepath.Rel(workingDir, absPath)
209209 if err != nil || strings.HasPrefix(relPath, "..") || filepath.IsAbs(relPath) {
210- return errResult("Access denied: path outside working directory"), nil
210+ return toolError("Access denied: path outside working directory"), nil
211211 }
212212
213213 // Dispatch to operation handler
@@ -217,13 +217,13 @@ func (h *EditHandler) Handle(ctx context.Context, arguments json.RawMessage) (*m
217217
218218 // Enforce read-before-edit: file must have been opened first
219219 if !h.tracker.WasRead(absPath) {
220- return errResult("You must Open the file before editing it. Use Open to read %s first — you need the line hashes.", args.File), nil
220+ return toolError("You must Open the file before editing it. Use Open to read %s first — you need the line hashes.", args.File), nil
221221 }
222222
223223 // All other ops require reading the existing file
224224 content, err := os.ReadFile(absPath)
225225 if err != nil {
226- return errResult("Failed to read file: %v", err), nil
226+ return toolError("Failed to read file: %v", err), nil
227227 }
228228 lines := strings.Split(string(content), "\n")
229229
@@ -238,11 +238,11 @@ func (h *EditHandler) Handle(ctx context.Context, arguments json.RawMessage) (*m
238238 }
239239
240240 if err != nil {
241- return errResult("%v", err), nil
241+ return toolError("%v", err), nil
242242 }
243243
244244 if err := os.WriteFile(absPath, []byte(result), 0600); err != nil {
245- return errResult("Failed to write file: %v", err), nil
245+ return toolError("Failed to write file: %v", err), nil
246246 }
247247
248248 // Show updated file in TUI editor
@@ -267,17 +267,17 @@ func (h *EditHandler) Handle(ctx context.Context, arguments json.RawMessage) (*m
267267 func (h *EditHandler) handleCreate(absPath, displayPath string, op *CreateOp) (*mcp.ToolResult, error) {
268268 // Fail if file already exists
269269 if _, err := os.Stat(absPath); err == nil {
270- return errResult("File already exists: %s (use replace/insert/delete to modify)", displayPath), nil
270+ return toolError("File already exists: %s (use replace/insert/delete to modify)", displayPath), nil
271271 }
272272
273273 // Create parent directories
274274 dir := filepath.Dir(absPath)
275275 if err := os.MkdirAll(dir, 0755); err != nil {
276- return errResult("Failed to create directories: %v", err), nil
276+ return toolError("Failed to create directories: %v", err), nil
277277 }
278278
279279 if err := os.WriteFile(absPath, []byte(op.Content), 0600); err != nil {
280- return errResult("Failed to create file: %v", err), nil
280+ return toolError("Failed to create file: %v", err), nil
281281 }
282282
283283 // Show in TUI
@@ -335,10 +335,3 @@ func applyDelete(lines []string, op *DeleteOp) (string, error) {
335335
336336 return strings.Join(newLines, "\n"), nil
337337 }
338-
339-func errResult(format string, args ...interface{}) *mcp.ToolResult {
340- return &mcp.ToolResult{
341- Content: []mcp.ContentBlock{{Type: "text", Text: fmt.Sprintf(format, args...)}},
342- IsError: true,
343- }
344-}
internal/mcp_tools/web.go+442−0
@@ -0,0 +1,442 @@
1+package mcp_tools
2+
3+import (
4+ "bytes"
5+ "context"
6+ "encoding/json"
7+ "fmt"
8+ "io"
9+ "net/http"
10+ "strings"
11+ "sync"
12+ "time"
13+
14+ "github.com/rs/zerolog/log"
15+ "github.com/xonecas/symb/internal/mcp"
16+ "golang.org/x/net/html"
17+)
18+
19+// cacheTTL is how long cached entries remain fresh.
20+const cacheTTL = 24 * time.Hour
21+
22+// cacheEntry stores a cached result with its timestamp.
23+type cacheEntry struct {
24+ result string
25+ createdAt time.Time
26+}
27+
28+// noSearchResults is the message returned when no search results are found.
29+const noSearchResults = "No results found."
30+
31+// WebCache is a shared in-memory cache for WebFetch and WebSearch results.
32+type WebCache struct {
33+ mu sync.RWMutex
34+ entries map[string]cacheEntry
35+}
36+
37+func newWebCache() *WebCache {
38+ return &WebCache{entries: make(map[string]cacheEntry)}
39+}
40+
41+// get returns the cached value and true if it exists and is fresh.
42+func (c *WebCache) get(key string) (string, bool) {
43+ c.mu.RLock()
44+ defer c.mu.RUnlock()
45+
46+ entry, ok := c.entries[key]
47+ if !ok {
48+ return "", false
49+ }
50+ if time.Since(entry.createdAt) > cacheTTL {
51+ return "", false
52+ }
53+ return entry.result, true
54+}
55+
56+// set stores a value in the cache.
57+func (c *WebCache) set(key, result string) {
58+ c.mu.Lock()
59+ defer c.mu.Unlock()
60+
61+ c.entries[key] = cacheEntry{result: result, createdAt: time.Now()}
62+}
63+
64+// --- WebFetch ---
65+
66+// WebFetchArgs represents arguments for the WebFetch tool.
67+type WebFetchArgs struct {
68+ URL string `json:"url"`
69+ MaxChars int `json:"max_chars,omitempty"`
70+}
71+
72+// NewWebFetchTool creates the WebFetch tool definition.
73+func NewWebFetchTool() mcp.Tool {
74+ schema := map[string]interface{}{
75+ "type": "object",
76+ "properties": map[string]interface{}{
77+ "url": map[string]interface{}{
78+ "type": "string",
79+ "description": "The URL to fetch.",
80+ },
81+ "max_chars": map[string]interface{}{
82+ "type": "integer",
83+ "description": "Maximum characters to return. Default: 10000",
84+ },
85+ },
86+ "required": []string{"url"},
87+ }
88+
89+ schemaJSON, _ := json.Marshal(schema)
90+
91+ return mcp.Tool{
92+ Name: "WebFetch",
93+ Description: "Fetch a URL and return its content as cleaned text (HTML tags, scripts, and styles stripped). Cached for 24 hours.",
94+ InputSchema: schemaJSON,
95+ }
96+}
97+
98+// MakeWebFetchHandler creates a handler for the WebFetch tool.
99+func MakeWebFetchHandler(cache *WebCache) mcp.ToolHandler {
100+ client := &http.Client{Timeout: 15 * time.Second}
101+
102+ return func(ctx context.Context, arguments json.RawMessage) (*mcp.ToolResult, error) {
103+ var args WebFetchArgs
104+ if err := json.Unmarshal(arguments, &args); err != nil {
105+ return toolError("Invalid arguments: %v", err), nil
106+ }
107+
108+ if args.URL == "" {
109+ return toolError("url is required"), nil
110+ }
111+ if args.MaxChars <= 0 {
112+ args.MaxChars = 10000
113+ }
114+
115+ // Check cache (keyed by raw URL).
116+ if cached, ok := cache.get(args.URL); ok {
117+ log.Debug().Str("url", args.URL).Msg("WebFetch cache hit")
118+ return toolText(truncate(cached, args.MaxChars)), nil
119+ }
120+
121+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, args.URL, nil)
122+ if err != nil {
123+ return toolError("Bad URL: %v", err), nil
124+ }
125+ req.Header.Set("User-Agent", "Symb/0.1")
126+ req.Header.Set("Accept", "text/html, text/plain;q=0.9, */*;q=0.5")
127+
128+ resp, err := client.Do(req)
129+ if err != nil {
130+ return toolError("Fetch failed: %v", err), nil
131+ }
132+ defer resp.Body.Close()
133+
134+ if resp.StatusCode >= 400 {
135+ return toolError("HTTP %d: %s", resp.StatusCode, resp.Status), nil
136+ }
137+
138+ // Cap read at 1MB to avoid blowing up memory.
139+ body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
140+ if err != nil {
141+ return toolError("Read failed: %v", err), nil
142+ }
143+
144+ contentType := resp.Header.Get("Content-Type")
145+ var text string
146+ if strings.Contains(contentType, "text/html") {
147+ text = extractText(body)
148+ } else {
149+ text = string(body)
150+ }
151+
152+ cache.set(args.URL, text)
153+ return toolText(truncate(text, args.MaxChars)), nil
154+ }
155+}
156+
157+// --- WebSearch ---
158+
159+// WebSearchArgs represents arguments for the WebSearch tool.
160+type WebSearchArgs struct {
161+ Query string `json:"query"`
162+ NumResults int `json:"num_results,omitempty"`
163+ Type string `json:"type,omitempty"`
164+ IncludeDomains []string `json:"include_domains,omitempty"`
165+}
166+
167+// exaSearchRequest is the request body for POST https://api.exa.ai/search.
168+type exaSearchRequest struct {
169+ Query string `json:"query"`
170+ Type string `json:"type"`
171+ NumResults int `json:"numResults"`
172+ Contents exaSearchContents `json:"contents"`
173+ IncludeDomains []string `json:"includeDomains,omitempty"`
174+}
175+
176+type exaSearchContents struct {
177+ Text exaTextOptions `json:"text"`
178+}
179+
180+type exaTextOptions struct {
181+ MaxCharacters int `json:"maxCharacters"`
182+}
183+
184+// exaSearchResponse is the response from Exa's /search endpoint.
185+type exaSearchResponse struct {
186+ Results []exaResult `json:"results"`
187+}
188+
189+type exaResult struct {
190+ Title string `json:"title"`
191+ URL string `json:"url"`
192+ Text string `json:"text"`
193+ PublishedDate string `json:"publishedDate,omitempty"`
194+}
195+
196+// NewWebSearchTool creates the WebSearch tool definition.
197+func NewWebSearchTool() mcp.Tool {
198+ schema := map[string]interface{}{
199+ "type": "object",
200+ "properties": map[string]interface{}{
201+ "query": map[string]interface{}{
202+ "type": "string",
203+ "description": "Search query.",
204+ },
205+ "num_results": map[string]interface{}{
206+ "type": "integer",
207+ "description": "Number of results to return. Default: 5",
208+ },
209+ "type": map[string]interface{}{
210+ "type": "string",
211+ "description": "Search type: \"auto\" (default), \"fast\", or \"deep\".",
212+ "enum": []string{"auto", "fast", "deep"},
213+ },
214+ "include_domains": map[string]interface{}{
215+ "type": "array",
216+ "items": map[string]interface{}{"type": "string"},
217+ "description": "Only include results from these domains.",
218+ },
219+ },
220+ "required": []string{"query"},
221+ }
222+
223+ schemaJSON, _ := json.Marshal(schema)
224+
225+ return mcp.Tool{
226+ Name: "WebSearch",
227+ Description: "Search the web using Exa AI. Use this to look up documentation, APIs, libraries, or current information. Cached for 24 hours.",
228+ InputSchema: schemaJSON,
229+ }
230+}
231+
232+const exaDefaultEndpoint = "https://api.exa.ai/search"
233+
234+// MakeWebSearchHandler creates a handler for the WebSearch tool.
235+// endpoint is the Exa API URL; pass "" to use the default (https://api.exa.ai/search).
236+func MakeWebSearchHandler(cache *WebCache, apiKey, endpoint string) mcp.ToolHandler {
237+ if endpoint == "" {
238+ endpoint = exaDefaultEndpoint
239+ }
240+ client := &http.Client{Timeout: 15 * time.Second}
241+
242+ return func(ctx context.Context, arguments json.RawMessage) (*mcp.ToolResult, error) {
243+ var args WebSearchArgs
244+ if err := json.Unmarshal(arguments, &args); err != nil {
245+ return toolError("Invalid arguments: %v", err), nil
246+ }
247+
248+ if args.Query == "" {
249+ return toolError("query is required"), nil
250+ }
251+ if apiKey == "" {
252+ return toolError("Exa AI API key not configured in credentials.json (providers.exa_ai.api_key)"), nil
253+ }
254+ if args.NumResults <= 0 {
255+ args.NumResults = 5
256+ }
257+ if args.Type == "" {
258+ args.Type = "auto"
259+ }
260+
261+ // Check cache (keyed by normalized query + params).
262+ cacheKey := fmt.Sprintf("search:%s:n=%d:t=%s:d=%s",
263+ strings.ToLower(strings.TrimSpace(args.Query)),
264+ args.NumResults, args.Type, strings.Join(args.IncludeDomains, ","))
265+ if cached, ok := cache.get(cacheKey); ok {
266+ log.Debug().Str("query", args.Query).Msg("WebSearch cache hit")
267+ return toolText(cached), nil
268+ }
269+
270+ body := exaSearchRequest{
271+ Query: args.Query,
272+ Type: args.Type,
273+ NumResults: args.NumResults,
274+ Contents: exaSearchContents{
275+ Text: exaTextOptions{MaxCharacters: 2000},
276+ },
277+ IncludeDomains: args.IncludeDomains,
278+ }
279+
280+ bodyJSON, err := json.Marshal(body)
281+ if err != nil {
282+ return toolError("Marshal failed: %v", err), nil
283+ }
284+
285+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(bodyJSON))
286+ if err != nil {
287+ return toolError("Request failed: %v", err), nil
288+ }
289+ req.Header.Set("Content-Type", "application/json")
290+ req.Header.Set("x-api-key", apiKey)
291+
292+ resp, err := client.Do(req)
293+ if err != nil {
294+ return toolError("Search failed: %v", err), nil
295+ }
296+ defer resp.Body.Close()
297+
298+ respBody, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
299+ if err != nil {
300+ return toolError("Read response failed: %v", err), nil
301+ }
302+
303+ if resp.StatusCode >= 400 {
304+ return toolError("Exa API error %d: %s", resp.StatusCode, string(respBody)), nil
305+ }
306+
307+ var exaResp exaSearchResponse
308+ if err := json.Unmarshal(respBody, &exaResp); err != nil {
309+ return toolError("Parse response failed: %v", err), nil
310+ }
311+
312+ result := formatSearchResults(exaResp.Results)
313+ cache.set(cacheKey, result)
314+ return toolText(result), nil
315+ }
316+}
317+
318+// NewWebCache creates a shared cache for web tools.
319+func NewWebCache() *WebCache {
320+ return newWebCache()
321+}
322+
323+// --- Helpers ---
324+
325+// formatSearchResults formats Exa results into readable text.
326+func formatSearchResults(results []exaResult) string {
327+ if len(results) == 0 {
328+ return noSearchResults
329+ }
330+
331+ var b strings.Builder
332+ b.WriteString(fmt.Sprintf("Found %d result(s):\n", len(results)))
333+ for i, r := range results {
334+ b.WriteString(fmt.Sprintf("\n--- %d. %s ---\n", i+1, r.Title))
335+ b.WriteString(fmt.Sprintf("URL: %s\n", r.URL))
336+ if r.PublishedDate != "" {
337+ b.WriteString(fmt.Sprintf("Published: %s\n", r.PublishedDate))
338+ }
339+ if r.Text != "" {
340+ b.WriteString(r.Text)
341+ b.WriteByte('\n')
342+ }
343+ }
344+ return b.String()
345+}
346+
347+// extractText parses HTML and returns visible text content.
348+// Strips script, style, and noscript elements.
349+func extractText(data []byte) string {
350+ tokenizer := html.NewTokenizer(bytes.NewReader(data))
351+ var b strings.Builder
352+ skip := 0 // nesting depth inside skip elements
353+
354+ for {
355+ tt := tokenizer.Next()
356+ switch tt {
357+ case html.ErrorToken:
358+ return collapseWhitespace(b.String())
359+
360+ case html.StartTagToken, html.SelfClosingTagToken:
361+ tn, _ := tokenizer.TagName()
362+ tag := string(tn)
363+ if tag == "script" || tag == "style" || tag == "noscript" {
364+ skip++
365+ }
366+ // Insert whitespace for block elements.
367+ if isBlockElement(tag) && b.Len() > 0 {
368+ b.WriteByte('\n')
369+ }
370+
371+ case html.EndTagToken:
372+ tn, _ := tokenizer.TagName()
373+ tag := string(tn)
374+ if tag == "script" || tag == "style" || tag == "noscript" {
375+ if skip > 0 {
376+ skip--
377+ }
378+ }
379+
380+ case html.TextToken:
381+ if skip == 0 {
382+ b.Write(tokenizer.Text())
383+ }
384+ }
385+ }
386+}
387+
388+// isBlockElement returns true for HTML elements that typically start a new line.
389+func isBlockElement(tag string) bool {
390+ switch tag {
391+ case "p", "div", "br", "h1", "h2", "h3", "h4", "h5", "h6",
392+ "li", "tr", "td", "th", "blockquote", "pre", "hr",
393+ "header", "footer", "section", "article", "nav", "main":
394+ return true
395+ }
396+ return false
397+}
398+
399+// collapseWhitespace trims each line and collapses multiple blank lines.
400+func collapseWhitespace(s string) string {
401+ lines := strings.Split(s, "\n")
402+ var out []string
403+ blanks := 0
404+ for _, line := range lines {
405+ trimmed := strings.TrimSpace(line)
406+ if trimmed == "" {
407+ blanks++
408+ if blanks <= 1 {
409+ out = append(out, "")
410+ }
411+ continue
412+ }
413+ blanks = 0
414+ out = append(out, trimmed)
415+ }
416+ return strings.TrimSpace(strings.Join(out, "\n"))
417+}
418+
419+// truncate cuts a string to maxChars (rune-safe).
420+func truncate(s string, maxChars int) string {
421+ runes := []rune(s)
422+ if len(runes) <= maxChars {
423+ return s
424+ }
425+ return string(runes[:maxChars]) + "\n\n[Truncated]"
426+}
427+
428+// toolError returns an error ToolResult.
429+func toolError(format string, args ...interface{}) *mcp.ToolResult {
430+ return &mcp.ToolResult{
431+ Content: []mcp.ContentBlock{{Type: "text", Text: fmt.Sprintf(format, args...)}},
432+ IsError: true,
433+ }
434+}
435+
436+// toolText returns a text ToolResult.
437+func toolText(text string) *mcp.ToolResult {
438+ return &mcp.ToolResult{
439+ Content: []mcp.ContentBlock{{Type: "text", Text: text}},
440+ IsError: false,
441+ }
442+}
internal/mcp_tools/web_test.go+297−0
@@ -0,0 +1,297 @@
1+package mcp_tools
2+
3+import (
4+ "context"
5+ "encoding/json"
6+ "net/http"
7+ "net/http/httptest"
8+ "testing"
9+ "time"
10+)
11+
12+func TestWebCache_GetSet(t *testing.T) {
13+ c := NewWebCache()
14+
15+ // Miss on empty cache.
16+ if _, ok := c.get("key"); ok {
17+ t.Fatal("expected cache miss")
18+ }
19+
20+ // Set and hit.
21+ c.set("key", "value")
22+ v, ok := c.get("key")
23+ if !ok || v != "value" {
24+ t.Fatalf("expected cache hit with 'value', got %q ok=%v", v, ok)
25+ }
26+}
27+
28+func TestWebCache_Expiry(t *testing.T) {
29+ c := NewWebCache()
30+ c.set("key", "value")
31+
32+ // Manually backdate the entry.
33+ c.mu.Lock()
34+ e := c.entries["key"]
35+ e.createdAt = time.Now().Add(-25 * time.Hour)
36+ c.entries["key"] = e
37+ c.mu.Unlock()
38+
39+ if _, ok := c.get("key"); ok {
40+ t.Fatal("expected cache miss for stale entry")
41+ }
42+}
43+
44+func TestExtractText(t *testing.T) {
45+ html := []byte(`<html><head><title>Hello</title><script>var x=1;</script><style>body{}</style></head>
46+<body><h1>Title</h1><p>Some <b>bold</b> text.</p><div>Another block</div></body></html>`)
47+
48+ text := extractText(html)
49+
50+ for _, want := range []string{"Title", "Some bold text.", "Another block"} {
51+ if !containsStr(text, want) {
52+ t.Errorf("expected text to contain %q, got:\n%s", want, text)
53+ }
54+ }
55+
56+ // Should NOT contain script/style content.
57+ for _, unwanted := range []string{"var x=1", "body{}"} {
58+ if containsStr(text, unwanted) {
59+ t.Errorf("expected text to NOT contain %q, got:\n%s", unwanted, text)
60+ }
61+ }
62+}
63+
64+func TestExtractText_PlainText(t *testing.T) {
65+ // No HTML at all — should pass through.
66+ text := extractText([]byte("just plain text"))
67+ if text != "just plain text" {
68+ t.Errorf("expected plain passthrough, got %q", text)
69+ }
70+}
71+
72+func TestTruncate(t *testing.T) {
73+ s := "hello world"
74+ if got := truncate(s, 100); got != s {
75+ t.Errorf("should not truncate, got %q", got)
76+ }
77+ if got := truncate(s, 5); got != "hello\n\n[Truncated]" {
78+ t.Errorf("unexpected truncation, got %q", got)
79+ }
80+}
81+
82+func TestCollapseWhitespace(t *testing.T) {
83+ input := " line1 \n\n\n\n line2 \n \n line3 "
84+ got := collapseWhitespace(input)
85+ want := "line1\n\nline2\n\nline3"
86+ if got != want {
87+ t.Errorf("collapseWhitespace:\ngot: %q\nwant: %q", got, want)
88+ }
89+}
90+
91+func TestWebFetchHandler_HTML(t *testing.T) {
92+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
93+ w.Header().Set("Content-Type", "text/html")
94+ w.Write([]byte(`<html><body><p>Hello from test</p></body></html>`))
95+ }))
96+ defer srv.Close()
97+
98+ cache := NewWebCache()
99+ handler := MakeWebFetchHandler(cache)
100+
101+ args, _ := json.Marshal(WebFetchArgs{URL: srv.URL})
102+ result, err := handler(context.Background(), args)
103+ if err != nil {
104+ t.Fatalf("unexpected error: %v", err)
105+ }
106+ if result.IsError {
107+ t.Fatalf("unexpected tool error: %s", result.Content[0].Text)
108+ }
109+ if !containsStr(result.Content[0].Text, "Hello from test") {
110+ t.Errorf("expected 'Hello from test' in result, got %q", result.Content[0].Text)
111+ }
112+
113+ // Second call should hit cache.
114+ result2, _ := handler(context.Background(), args)
115+ if result2.Content[0].Text != result.Content[0].Text {
116+ t.Error("expected identical cached result")
117+ }
118+}
119+
120+func TestWebFetchHandler_PlainText(t *testing.T) {
121+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
122+ w.Header().Set("Content-Type", "text/plain")
123+ w.Write([]byte("plain content"))
124+ }))
125+ defer srv.Close()
126+
127+ cache := NewWebCache()
128+ handler := MakeWebFetchHandler(cache)
129+
130+ args, _ := json.Marshal(WebFetchArgs{URL: srv.URL})
131+ result, err := handler(context.Background(), args)
132+ if err != nil {
133+ t.Fatalf("unexpected error: %v", err)
134+ }
135+ if result.Content[0].Text != "plain content" {
136+ t.Errorf("expected 'plain content', got %q", result.Content[0].Text)
137+ }
138+}
139+
140+func TestWebFetchHandler_Error(t *testing.T) {
141+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
142+ w.WriteHeader(http.StatusNotFound)
143+ }))
144+ defer srv.Close()
145+
146+ cache := NewWebCache()
147+ handler := MakeWebFetchHandler(cache)
148+
149+ args, _ := json.Marshal(WebFetchArgs{URL: srv.URL})
150+ result, _ := handler(context.Background(), args)
151+ if !result.IsError {
152+ t.Fatal("expected error result for 404")
153+ }
154+}
155+
156+func TestWebFetchHandler_MissingURL(t *testing.T) {
157+ cache := NewWebCache()
158+ handler := MakeWebFetchHandler(cache)
159+
160+ args, _ := json.Marshal(WebFetchArgs{})
161+ result, _ := handler(context.Background(), args)
162+ if !result.IsError {
163+ t.Fatal("expected error for missing URL")
164+ }
165+}
166+
167+func TestWebSearchHandler_MissingKey(t *testing.T) {
168+ cache := NewWebCache()
169+ handler := MakeWebSearchHandler(cache, "", "")
170+
171+ args, _ := json.Marshal(WebSearchArgs{Query: "test"})
172+ result, _ := handler(context.Background(), args)
173+ if !result.IsError {
174+ t.Fatal("expected error for missing API key")
175+ }
176+ if !containsStr(result.Content[0].Text, "not configured") {
177+ t.Errorf("expected 'not configured' in error, got %q", result.Content[0].Text)
178+ }
179+}
180+
181+func TestWebSearchHandler_MissingQuery(t *testing.T) {
182+ cache := NewWebCache()
183+ handler := MakeWebSearchHandler(cache, "fake-key", "")
184+
185+ args, _ := json.Marshal(WebSearchArgs{})
186+ result, _ := handler(context.Background(), args)
187+ if !result.IsError {
188+ t.Fatal("expected error for missing query")
189+ }
190+}
191+
192+func TestWebSearchHandler_Success(t *testing.T) {
193+ srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
194+ // Verify auth header.
195+ if r.Header.Get("x-api-key") != "test-key" {
196+ w.WriteHeader(http.StatusUnauthorized)
197+ return
198+ }
199+
200+ // Verify request body.
201+ var req exaSearchRequest
202+ if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
203+ w.WriteHeader(http.StatusBadRequest)
204+ return
205+ }
206+
207+ resp := exaSearchResponse{
208+ Results: []exaResult{
209+ {Title: "Result 1", URL: "https://example.com/1", Text: "Some text"},
210+ {Title: "Result 2", URL: "https://example.com/2", Text: "More text", PublishedDate: "2025-01-01"},
211+ },
212+ }
213+ json.NewEncoder(w).Encode(resp)
214+ }))
215+ defer srv.Close()
216+
217+ t.Run("integration", func(t *testing.T) {
218+ cache := NewWebCache()
219+ handler := MakeWebSearchHandler(cache, "test-key", srv.URL)
220+
221+ args, _ := json.Marshal(WebSearchArgs{Query: "test query", NumResults: 2})
222+ result, err := handler(context.Background(), args)
223+ if err != nil {
224+ t.Fatalf("unexpected error: %v", err)
225+ }
226+ if result.IsError {
227+ t.Fatalf("unexpected tool error: %s", result.Content[0].Text)
228+ }
229+ if !containsStr(result.Content[0].Text, "Result 1") {
230+ t.Errorf("expected 'Result 1' in output, got %q", result.Content[0].Text)
231+ }
232+ if !containsStr(result.Content[0].Text, "Published: 2025-01-01") {
233+ t.Errorf("expected published date in output, got %q", result.Content[0].Text)
234+ }
235+
236+ // Second call should hit cache.
Cut short: this commit's diff is over 1000 lines.