@@ -1,360 +0,0 @@
1-package mcptools
2-
3-import (
4- "bytes"
5- "context"
6- "encoding/json"
7- "fmt"
8- "io"
9- "net/http"
10- "strings"
11- "time"
12-
13- "github.com/rs/zerolog/log"
14- "github.com/xonecas/symb/internal/mcp"
15- "github.com/xonecas/symb/internal/store"
16- "golang.org/x/net/html"
17-)
18-
19-// noSearchResults is the message returned when no search results are found.
20-const noSearchResults = "No results found."
21-
22-// --- WebFetch ---
23-
24-// WebFetchArgs represents arguments for the WebFetch tool.
25-type WebFetchArgs struct {
26- URL string `json:"url"`
27- MaxChars int `json:"max_chars,omitempty"`
28-}
29-
30-// NewWebFetchTool creates the WebFetch tool definition.
31-func NewWebFetchTool() mcp.Tool {
32- return mcp.Tool{
33- Name: "WebFetch",
34- Description: "Fetch a URL and return its content as cleaned text (HTML tags, scripts, and styles stripped). Results are cached.",
35- InputSchema: json.RawMessage(`{
36- "type": "object",
37- "properties": {
38- "url": {"type": "string", "description": "The URL to fetch."},
39- "max_chars": {"type": "integer", "description": "Maximum characters to return. Default: 10000"}
40- },
41- "required": ["url"]
42- }`),
43- }
44-}
45-
46-// MakeWebFetchHandler creates a handler for the WebFetch tool.
47-func MakeWebFetchHandler(cache *store.Cache) mcp.ToolHandler {
48- client := &http.Client{Timeout: 15 * time.Second}
49-
50- return func(ctx context.Context, arguments json.RawMessage) (*mcp.ToolResult, error) {
51- var args WebFetchArgs
52- if err := json.Unmarshal(arguments, &args); err != nil {
53- return toolError("Invalid arguments: %v", err), nil
54- }
55-
56- if args.URL == "" {
57- return toolError("url is required"), nil
58- }
59- if args.MaxChars <= 0 {
60- args.MaxChars = 10000
61- }
62-
63- // Check cache (keyed by URL).
64- if cached, ok := cache.GetFetch(args.URL); ok {
65- log.Debug().Str("url", args.URL).Msg("WebFetch cache hit")
66- return toolText(truncate(cached, args.MaxChars)), nil
67- }
68-
69- req, err := http.NewRequestWithContext(ctx, http.MethodGet, args.URL, nil)
70- if err != nil {
71- return toolError("Bad URL: %v", err), nil
72- }
73- req.Header.Set("User-Agent", "Symb/0.1")
74- req.Header.Set("Accept", "text/html, text/plain;q=0.9, */*;q=0.5")
75-
76- resp, err := client.Do(req)
77- if err != nil {
78- return toolError("Fetch failed: %v", err), nil
79- }
80- defer resp.Body.Close()
81-
82- if resp.StatusCode >= 400 {
83- return toolError("HTTP %d: %s", resp.StatusCode, resp.Status), nil
84- }
85-
86- // Cap read at 1MB to avoid blowing up memory.
87- body, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
88- if err != nil {
89- return toolError("Read failed: %v", err), nil
90- }
91-
92- contentType := resp.Header.Get("Content-Type")
93- var text string
94- if strings.Contains(contentType, "text/html") {
95- text = extractText(body)
96- } else {
97- text = string(body)
98- }
99-
100- cache.SetFetch(args.URL, text)
101- return toolText(truncate(text, args.MaxChars)), nil
102- }
103-}
104-
105-// --- WebSearch ---
106-
107-// WebSearchArgs represents arguments for the WebSearch tool.
108-type WebSearchArgs struct {
109- Query string `json:"query"`
110- NumResults int `json:"num_results,omitempty"`
111- Type string `json:"type,omitempty"`
112- IncludeDomains []string `json:"include_domains,omitempty"`
113-}
114-
115-// exaSearchRequest is the request body for POST https://api.exa.ai/search.
116-type exaSearchRequest struct {
117- Query string `json:"query"`
118- Type string `json:"type"`
119- NumResults int `json:"numResults"`
120- Contents exaSearchContents `json:"contents"`
121- IncludeDomains []string `json:"includeDomains,omitempty"`
122-}
123-
124-type exaSearchContents struct {
125- Text exaTextOptions `json:"text"`
126-}
127-
128-type exaTextOptions struct {
129- MaxCharacters int `json:"maxCharacters"`
130-}
131-
132-// exaSearchResponse is the response from Exa's /search endpoint.
133-type exaSearchResponse struct {
134- Results []exaResult `json:"results"`
135-}
136-
137-type exaResult struct {
138- Title string `json:"title"`
139- URL string `json:"url"`
140- Text string `json:"text"`
141- PublishedDate string `json:"publishedDate,omitempty"`
142-}
143-
144-// NewWebSearchTool creates the WebSearch tool definition.
145-func NewWebSearchTool() mcp.Tool {
146- return mcp.Tool{
147- Name: "WebSearch",
148- Description: "Search the web using Exa AI. Use this to look up documentation, APIs, libraries, or current information. Results are cached.",
149- InputSchema: json.RawMessage(`{
150- "type": "object",
151- "properties": {
152- "query": {"type": "string", "description": "Search query."},
153- "num_results": {"type": "integer", "description": "Number of results to return. Default: 5"},
154- "type": {"type": "string", "description": "Search type: \"auto\" (default), \"fast\", or \"deep\".", "enum": ["auto", "fast", "deep"]},
155- "include_domains": {"type": "array", "items": {"type": "string"}, "description": "Only include results from these domains."}
156- },
157- "required": ["query"]
158- }`),
159- }
160-}
161-
162-const exaDefaultEndpoint = "https://api.exa.ai/search"
163-
164-// MakeWebSearchHandler creates a handler for the WebSearch tool.
165-// endpoint is the Exa API URL; pass "" to use the default (https://api.exa.ai/search).
166-func MakeWebSearchHandler(cache *store.Cache, apiKey, endpoint string) mcp.ToolHandler {
167- if endpoint == "" {
168- endpoint = exaDefaultEndpoint
169- }
170- client := &http.Client{Timeout: 15 * time.Second}
171-
172- return func(ctx context.Context, arguments json.RawMessage) (*mcp.ToolResult, error) {
173- var args WebSearchArgs
174- if err := json.Unmarshal(arguments, &args); err != nil {
175- return toolError("Invalid arguments: %v", err), nil
176- }
177-
178- if args.Query == "" {
179- return toolError("query is required"), nil
180- }
181- if apiKey == "" {
182- return toolError("Exa AI API key not configured in credentials.json (providers.exa_ai.api_key)"), nil
183- }
184- if args.NumResults <= 0 {
185- args.NumResults = 5
186- }
187- if args.Type == "" {
188- args.Type = "auto"
189- }
190-
191- // Build exact cache key including params so different num_results/type
192- // don't return wrong cached results.
193- exactKey := fmt.Sprintf("%s|n=%d|t=%s|d=%s",
194- args.Query, args.NumResults, args.Type,
195- strings.Join(args.IncludeDomains, ","))
196-
197- // Check exact cache hit first (query + params).
198- if cached, ok := cache.GetSearch(exactKey); ok {
199- log.Debug().Str("query", args.Query).Msg("WebSearch exact cache hit")
200- return toolText(cached), nil
201- }
202-
203- // Search cached result content for query keywords — avoids API call
204- // if the answer already exists in a previously cached result.
205- if cached, ok := cache.SearchCachedContent(args.Query); ok {
206- log.Debug().Str("query", args.Query).Msg("WebSearch content cache hit")
207- return toolText(cached), nil
208- }
209-
210- body := exaSearchRequest{
211- Query: args.Query,
212- Type: args.Type,
213- NumResults: args.NumResults,
214- Contents: exaSearchContents{
215- Text: exaTextOptions{MaxCharacters: 2000},
216- },
217- IncludeDomains: args.IncludeDomains,
218- }
219-
220- bodyJSON, err := json.Marshal(body)
221- if err != nil {
222- return toolError("Marshal failed: %v", err), nil
223- }
224-
225- req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(bodyJSON))
226- if err != nil {
227- return toolError("Request failed: %v", err), nil
228- }
229- req.Header.Set("Content-Type", "application/json")
230- req.Header.Set("x-api-key", apiKey)
231-
232- resp, err := client.Do(req)
233- if err != nil {
234- return toolError("Search failed: %v", err), nil
235- }
236- defer resp.Body.Close()
237-
238- respBody, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
239- if err != nil {
240- return toolError("Read response failed: %v", err), nil
241- }
242-
243- if resp.StatusCode >= 400 {
244- return toolError("Exa API error %d: %s", resp.StatusCode, string(respBody)), nil
245- }
246-
247- var exaResp exaSearchResponse
248- if err := json.Unmarshal(respBody, &exaResp); err != nil {
249- return toolError("Parse response failed: %v", err), nil
250- }
251-
252- result := formatSearchResults(exaResp.Results)
253- cache.SetSearch(exactKey, result)
254- return toolText(result), nil
255- }
256-}
257-
258-// --- Helpers ---
259-
260-// formatSearchResults formats Exa results into readable text.
261-func formatSearchResults(results []exaResult) string {
262- if len(results) == 0 {
263- return noSearchResults
264- }
265-
266- var b strings.Builder
267- b.WriteString(fmt.Sprintf("Found %d result(s):\n", len(results)))
268- for i, r := range results {
269- b.WriteString(fmt.Sprintf("\n--- %d. %s ---\n", i+1, r.Title))
270- b.WriteString(fmt.Sprintf("URL: %s\n", r.URL))
271- if r.PublishedDate != "" {
272- b.WriteString(fmt.Sprintf("Published: %s\n", r.PublishedDate))
273- }
274- if r.Text != "" {
275- b.WriteString(r.Text)
276- b.WriteByte('\n')
277- }
278- }
279- return b.String()
280-}
281-
282-// isSkipTag returns true for tags whose content should be suppressed.
283-func isSkipTag(tag string) bool {
284- return tag == "script" || tag == "style" || tag == "noscript"
285-}
286-
287-// extractText parses HTML and returns visible text content.
288-// Strips script, style, and noscript elements.
289-func extractText(data []byte) string {
290- tokenizer := html.NewTokenizer(bytes.NewReader(data))
291- var b strings.Builder
292- skip := 0
293-
294- for {
295- tt := tokenizer.Next()
296- if tt == html.ErrorToken {
297- return collapseWhitespace(b.String())
298- }
299- tn, _ := tokenizer.TagName()
300- tag := string(tn)
301-
302- switch tt {
303- case html.StartTagToken, html.SelfClosingTagToken:
304- if isSkipTag(tag) {
305- skip++
306- }
307- if isBlockElement(tag) && b.Len() > 0 {
308- b.WriteByte('\n')
309- }
310- case html.EndTagToken:
311- if isSkipTag(tag) && skip > 0 {
312- skip--
313- }
314- case html.TextToken:
315- if skip == 0 {
316- b.Write(tokenizer.Text())
317- }
318- }
319- }
320-}
321-
322-// isBlockElement returns true for HTML elements that typically start a new line.
323-func isBlockElement(tag string) bool {
324- switch tag {
325- case "p", "div", "br", "h1", "h2", "h3", "h4", "h5", "h6",
326- "li", "tr", "td", "th", "blockquote", "pre", "hr",
327- "header", "footer", "section", "article", "nav", "main":
328- return true
329- }
330- return false
331-}
332-
333-// collapseWhitespace trims each line and collapses multiple blank lines.
334-func collapseWhitespace(s string) string {
335- lines := strings.Split(s, "\n")
336- var out []string
337- blanks := 0
338- for _, line := range lines {
339- trimmed := strings.TrimSpace(line)
340- if trimmed == "" {
341- blanks++
342- if blanks <= 1 {
343- out = append(out, "")
344- }
345- continue
346- }
347- blanks = 0
348- out = append(out, trimmed)
349- }
350- return strings.TrimSpace(strings.Join(out, "\n"))
351-}
352-
353-// truncate cuts a string to maxChars (rune-safe).
354-func truncate(s string, maxChars int) string {
355- runes := []rune(s)
356- if len(runes) <= maxChars {
357- return s
358- }
359- return string(runes[:maxChars]) + "\n\n[Truncated]"
360-}