@@ -0,0 +1,442 @@
1+package mcp_tools
2+
3+import (
4+ "bytes"
5+ "context"
6+ "encoding/json"
7+ "fmt"
8+ "io"
9+ "net/http"
10+ "strings"
11+ "sync"
12+ "time"
13+
14+ "github.com/rs/zerolog/log"
15+ "github.com/xonecas/symb/internal/mcp"
16+ "golang.org/x/net/html"
17+)
18+
19+// cacheTTL is how long cached entries remain fresh.
20+const cacheTTL = 24 * time.Hour
21+
22+// cacheEntry stores a cached result with its timestamp.
23+type cacheEntry struct {
24+ result string
25+ createdAt time.Time
26+}
27+
28+// noSearchResults is the message returned when no search results are found.
29+const noSearchResults = "No results found."
30+
31+// WebCache is a shared in-memory cache for WebFetch and WebSearch results.
32+type WebCache struct {
33+ mu sync.RWMutex
34+ entries map[string]cacheEntry
35+}
36+
37+func newWebCache() *WebCache {
38+ return &WebCache{entries: make(map[string]cacheEntry)}
39+}
40+
41+// get returns the cached value and true if it exists and is fresh.
42+func (c *WebCache) get(key string) (string, bool) {
43+ c.mu.RLock()
44+ defer c.mu.RUnlock()
45+
46+ entry, ok := c.entries[key]
47+ if !ok {
48+ return "", false
49+ }
50+ if time.Since(entry.createdAt) > cacheTTL {
51+ return "", false
52+ }
53+ return entry.result, true
54+}
55+
56+// set stores a value in the cache.
57+func (c *WebCache) set(key, result string) {
58+ c.mu.Lock()
59+ defer c.mu.Unlock()
60+
61+ c.entries[key] = cacheEntry{result: result, createdAt: time.Now()}
62+}
63+
64+// --- WebFetch ---
65+
66+// WebFetchArgs represents arguments for the WebFetch tool.
67+type WebFetchArgs struct {
68+ URL string `json:"url"`
69+ MaxChars int `json:"max_chars,omitempty"`
70+}
71+
72+// NewWebFetchTool creates the WebFetch tool definition.
73+func NewWebFetchTool() mcp.Tool {
74+ schema := map[string]interface{}{
75+ "type": "object",
76+ "properties": map[string]interface{}{
77+ "url": map[string]interface{}{
78+ "type": "string",
79+ "description": "The URL to fetch.",
80+ },
81+ "max_chars": map[string]interface{}{
82+ "type": "integer",
83+ "description": "Maximum characters to return. Default: 10000",
84+ },
85+ },
86+ "required": []string{"url"},
87+ }
88+
89+ schemaJSON, _ := json.Marshal(schema)
90+
91+ return mcp.Tool{
92+ Name: "WebFetch",
93+ Description: "Fetch a URL and return its content as cleaned text (HTML tags, scripts, and styles stripped). Cached for 24 hours.",
94+ InputSchema: schemaJSON,
95+ }
96+}
97+
98+// MakeWebFetchHandler creates a handler for the WebFetch tool.
99+func MakeWebFetchHandler(cache *WebCache) mcp.ToolHandler {
100+ client := &http.Client{Timeout: 15 * time.Second}
101+
102+ return func(ctx context.Context, arguments json.RawMessage) (*mcp.ToolResult, error) {
103+ var args WebFetchArgs
104+ if err := json.Unmarshal(arguments, &args); err != nil {
105+ return toolError("Invalid arguments: %v", err), nil
106+ }
107+
108+ if args.URL == "" {
109+ return toolError("url is required"), nil
110+ }
111+ if args.MaxChars <= 0 {
112+ args.MaxChars = 10000
113+ }
114+
115+ // Check cache (keyed by raw URL).
116+ if cached, ok := cache.get(args.URL); ok {
117+ log.Debug().Str("url", args.URL).Msg("WebFetch cache hit")
118+ return toolText(truncate(cached, args.MaxChars)), nil
119+ }
120+
121+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, args.URL, nil)
122+ if err != nil {
123+ return toolError("Bad URL: %v", err), nil
124+ }
125+ req.Header.Set("User-Agent", "Symb/0.1")
126+ req.Header.Set("Accept", "text/html, text/plain;q=0.9, */*;q=0.5")
127+
128+ resp, err := client.Do(req)
129+ if err != nil {
130+ return toolError("Fetch failed: %v", err), nil
131+ }
132+ defer resp.Body.Close()
133+
134+ if resp.StatusCode >= 400 {
135+ return toolError("HTTP %d: %s", resp.StatusCode, resp.Status), nil
136+ }
137+
138+ // Cap read at 1MB to avoid blowing up memory.
139+ body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
140+ if err != nil {
141+ return toolError("Read failed: %v", err), nil
142+ }
143+
144+ contentType := resp.Header.Get("Content-Type")
145+ var text string
146+ if strings.Contains(contentType, "text/html") {
147+ text = extractText(body)
148+ } else {
149+ text = string(body)
150+ }
151+
152+ cache.set(args.URL, text)
153+ return toolText(truncate(text, args.MaxChars)), nil
154+ }
155+}
156+
157+// --- WebSearch ---
158+
159+// WebSearchArgs represents arguments for the WebSearch tool.
160+type WebSearchArgs struct {
161+ Query string `json:"query"`
162+ NumResults int `json:"num_results,omitempty"`
163+ Type string `json:"type,omitempty"`
164+ IncludeDomains []string `json:"include_domains,omitempty"`
165+}
166+
167+// exaSearchRequest is the request body for POST https://api.exa.ai/search.
168+type exaSearchRequest struct {
169+ Query string `json:"query"`
170+ Type string `json:"type"`
171+ NumResults int `json:"numResults"`
172+ Contents exaSearchContents `json:"contents"`
173+ IncludeDomains []string `json:"includeDomains,omitempty"`
174+}
175+
176+type exaSearchContents struct {
177+ Text exaTextOptions `json:"text"`
178+}
179+
180+type exaTextOptions struct {
181+ MaxCharacters int `json:"maxCharacters"`
182+}
183+
184+// exaSearchResponse is the response from Exa's /search endpoint.
185+type exaSearchResponse struct {
186+ Results []exaResult `json:"results"`
187+}
188+
189+type exaResult struct {
190+ Title string `json:"title"`
191+ URL string `json:"url"`
192+ Text string `json:"text"`
193+ PublishedDate string `json:"publishedDate,omitempty"`
194+}
195+
196+// NewWebSearchTool creates the WebSearch tool definition.
197+func NewWebSearchTool() mcp.Tool {
198+ schema := map[string]interface{}{
199+ "type": "object",
200+ "properties": map[string]interface{}{
201+ "query": map[string]interface{}{
202+ "type": "string",
203+ "description": "Search query.",
204+ },
205+ "num_results": map[string]interface{}{
206+ "type": "integer",
207+ "description": "Number of results to return. Default: 5",
208+ },
209+ "type": map[string]interface{}{
210+ "type": "string",
211+ "description": "Search type: \"auto\" (default), \"fast\", or \"deep\".",
212+ "enum": []string{"auto", "fast", "deep"},
213+ },
214+ "include_domains": map[string]interface{}{
215+ "type": "array",
216+ "items": map[string]interface{}{"type": "string"},
217+ "description": "Only include results from these domains.",
218+ },
219+ },
220+ "required": []string{"query"},
221+ }
222+
223+ schemaJSON, _ := json.Marshal(schema)
224+
225+ return mcp.Tool{
226+ Name: "WebSearch",
227+ Description: "Search the web using Exa AI. Use this to look up documentation, APIs, libraries, or current information. Cached for 24 hours.",
228+ InputSchema: schemaJSON,
229+ }
230+}
231+
232+const exaDefaultEndpoint = "https://api.exa.ai/search"
233+
234+// MakeWebSearchHandler creates a handler for the WebSearch tool.
235+// endpoint is the Exa API URL; pass "" to use the default (https://api.exa.ai/search).
236+func MakeWebSearchHandler(cache *WebCache, apiKey, endpoint string) mcp.ToolHandler {
237+ if endpoint == "" {
238+ endpoint = exaDefaultEndpoint
239+ }
240+ client := &http.Client{Timeout: 15 * time.Second}
241+
242+ return func(ctx context.Context, arguments json.RawMessage) (*mcp.ToolResult, error) {
243+ var args WebSearchArgs
244+ if err := json.Unmarshal(arguments, &args); err != nil {
245+ return toolError("Invalid arguments: %v", err), nil
246+ }
247+
248+ if args.Query == "" {
249+ return toolError("query is required"), nil
250+ }
251+ if apiKey == "" {
252+ return toolError("Exa AI API key not configured in credentials.json (providers.exa_ai.api_key)"), nil
253+ }
254+ if args.NumResults <= 0 {
255+ args.NumResults = 5
256+ }
257+ if args.Type == "" {
258+ args.Type = "auto"
259+ }
260+
261+ // Check cache (keyed by normalized query + params).
262+ cacheKey := fmt.Sprintf("search:%s:n=%d:t=%s:d=%s",
263+ strings.ToLower(strings.TrimSpace(args.Query)),
264+ args.NumResults, args.Type, strings.Join(args.IncludeDomains, ","))
265+ if cached, ok := cache.get(cacheKey); ok {
266+ log.Debug().Str("query", args.Query).Msg("WebSearch cache hit")
267+ return toolText(cached), nil
268+ }
269+
270+ body := exaSearchRequest{
271+ Query: args.Query,
272+ Type: args.Type,
273+ NumResults: args.NumResults,
274+ Contents: exaSearchContents{
275+ Text: exaTextOptions{MaxCharacters: 2000},
276+ },
277+ IncludeDomains: args.IncludeDomains,
278+ }
279+
280+ bodyJSON, err := json.Marshal(body)
281+ if err != nil {
282+ return toolError("Marshal failed: %v", err), nil
283+ }
284+
285+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(bodyJSON))
286+ if err != nil {
287+ return toolError("Request failed: %v", err), nil
288+ }
289+ req.Header.Set("Content-Type", "application/json")
290+ req.Header.Set("x-api-key", apiKey)
291+
292+ resp, err := client.Do(req)
293+ if err != nil {
294+ return toolError("Search failed: %v", err), nil
295+ }
296+ defer resp.Body.Close()
297+
298+ respBody, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
299+ if err != nil {
300+ return toolError("Read response failed: %v", err), nil
301+ }
302+
303+ if resp.StatusCode >= 400 {
304+ return toolError("Exa API error %d: %s", resp.StatusCode, string(respBody)), nil
305+ }
306+
307+ var exaResp exaSearchResponse
308+ if err := json.Unmarshal(respBody, &exaResp); err != nil {
309+ return toolError("Parse response failed: %v", err), nil
310+ }
311+
312+ result := formatSearchResults(exaResp.Results)
313+ cache.set(cacheKey, result)
314+ return toolText(result), nil
315+ }
316+}
317+
318+// NewWebCache creates a shared cache for web tools.
319+func NewWebCache() *WebCache {
320+ return newWebCache()
321+}
322+
323+// --- Helpers ---
324+
325+// formatSearchResults formats Exa results into readable text.
326+func formatSearchResults(results []exaResult) string {
327+ if len(results) == 0 {
328+ return noSearchResults
329+ }
330+
331+ var b strings.Builder
332+ b.WriteString(fmt.Sprintf("Found %d result(s):\n", len(results)))
333+ for i, r := range results {
334+ b.WriteString(fmt.Sprintf("\n--- %d. %s ---\n", i+1, r.Title))
335+ b.WriteString(fmt.Sprintf("URL: %s\n", r.URL))
336+ if r.PublishedDate != "" {
337+ b.WriteString(fmt.Sprintf("Published: %s\n", r.PublishedDate))
338+ }
339+ if r.Text != "" {
340+ b.WriteString(r.Text)
341+ b.WriteByte('\n')
342+ }
343+ }
344+ return b.String()
345+}
346+
347+// extractText parses HTML and returns visible text content.
348+// Strips script, style, and noscript elements.
349+func extractText(data []byte) string {
350+ tokenizer := html.NewTokenizer(bytes.NewReader(data))
351+ var b strings.Builder
352+ skip := 0 // nesting depth inside skip elements
353+
354+ for {
355+ tt := tokenizer.Next()
356+ switch tt {
357+ case html.ErrorToken:
358+ return collapseWhitespace(b.String())
359+
360+ case html.StartTagToken, html.SelfClosingTagToken:
361+ tn, _ := tokenizer.TagName()
362+ tag := string(tn)
363+ if tag == "script" || tag == "style" || tag == "noscript" {
364+ skip++
365+ }
366+ // Insert whitespace for block elements.
367+ if isBlockElement(tag) && b.Len() > 0 {
368+ b.WriteByte('\n')
369+ }
370+
371+ case html.EndTagToken:
372+ tn, _ := tokenizer.TagName()
373+ tag := string(tn)
374+ if tag == "script" || tag == "style" || tag == "noscript" {
375+ if skip > 0 {
376+ skip--
377+ }
378+ }
379+
380+ case html.TextToken:
381+ if skip == 0 {
382+ b.Write(tokenizer.Text())
383+ }
384+ }
385+ }
386+}
387+
388+// isBlockElement returns true for HTML elements that typically start a new line.
389+func isBlockElement(tag string) bool {
390+ switch tag {
391+ case "p", "div", "br", "h1", "h2", "h3", "h4", "h5", "h6",
392+ "li", "tr", "td", "th", "blockquote", "pre", "hr",
393+ "header", "footer", "section", "article", "nav", "main":
394+ return true
395+ }
396+ return false
397+}
398+
399+// collapseWhitespace trims each line and collapses multiple blank lines.
400+func collapseWhitespace(s string) string {
401+ lines := strings.Split(s, "\n")
402+ var out []string
403+ blanks := 0
404+ for _, line := range lines {
405+ trimmed := strings.TrimSpace(line)
406+ if trimmed == "" {
407+ blanks++
408+ if blanks <= 1 {
409+ out = append(out, "")
410+ }
411+ continue
412+ }
413+ blanks = 0
414+ out = append(out, trimmed)
415+ }
416+ return strings.TrimSpace(strings.Join(out, "\n"))
417+}
418+
419+// truncate cuts a string to maxChars (rune-safe).
420+func truncate(s string, maxChars int) string {
421+ runes := []rune(s)
422+ if len(runes) <= maxChars {
423+ return s
424+ }
425+ return string(runes[:maxChars]) + "\n\n[Truncated]"
426+}
427+
428+// toolError returns an error ToolResult.
429+func toolError(format string, args ...interface{}) *mcp.ToolResult {
430+ return &mcp.ToolResult{
431+ Content: []mcp.ContentBlock{{Type: "text", Text: fmt.Sprintf(format, args...)}},
432+ IsError: true,
433+ }
434+}
435+
436+// toolText returns a text ToolResult.
437+func toolText(text string) *mcp.ToolResult {
438+ return &mcp.ToolResult{
439+ Content: []mcp.ContentBlock{{Type: "text", Text: text}},
440+ IsError: false,
441+ }
442+}