xonecas.io
62159
Sy

symb / files

symb / internal / llm / loop.go

12.3 KB
1// Package llm implements the LLM interaction loop with tool calling support.
2package llm
3
4import (
5	"context"
6	"encoding/json"
7	"fmt"
8	"strings"
9	"time"
10
11	"github.com/rs/zerolog/log"
12	"github.com/xonecas/symb/internal/mcp"
13	"github.com/xonecas/symb/internal/provider"
14)
15
16const (
17	// MaxDepth is the maximum recursion depth for sub-agents.
18	// Matches subagent.MaxSubAgentDepth to prevent import cycle.
19	MaxDepth = 1
20)
21
22// MessageCallback is called when a complete message should be added to history.
23type MessageCallback func(msg provider.Message)
24
25// DeltaCallback is called for each streaming event (content/reasoning deltas).
26type DeltaCallback func(evt provider.StreamEvent)
27
28// ToolCallCallback is called when tool calls are about to be executed.
29type ToolCallCallback func()
30
31// UsageCallback is called with accumulated token usage after each LLM call.
32type UsageCallback func(inputTokens, outputTokens int)
33
34// ScratchpadReader provides read access to the agent's working plan.
35type ScratchpadReader interface {
36	Content() string
37}
38
39// ProcessTurnOptions holds configuration for processing a turn.
40type ProcessTurnOptions struct {
41	Provider      provider.Provider
42	Proxy         *mcp.Proxy
43	Tools         []mcp.Tool
44	History       []provider.Message
45	OnMessage     MessageCallback
46	OnDelta       DeltaCallback    // Optional: called for each stream event
47	OnToolCall    ToolCallCallback // Optional: called before executing tool calls
48	OnUsage       UsageCallback    // Optional: called with token usage after each LLM call
49	Scratchpad    ScratchpadReader // Optional: agent plan injected at context tail
50	MaxToolRounds int
51	Depth         int // Recursion depth (0=root agent, 1=sub-agent)
52}
53
54// streamAndCollect runs one LLM call: streams events, collects the response,
55// reports usage, and returns the ChatResponse.
56func streamAndCollect(ctx context.Context, opts *ProcessTurnOptions, tools []provider.Tool) (*provider.ChatResponse, error) {
57	const maxEmptyRetries = 1
58
59	for attempt := 0; attempt <= maxEmptyRetries; attempt++ {
60		stream, err := opts.Provider.ChatStream(ctx, opts.History, tools)
61		if err != nil {
62			return nil, err
63		}
64		resp, err := collectWithDeltas(stream, opts.OnDelta)
65		if err != nil {
66			return nil, err
67		}
68		if opts.OnUsage != nil && (resp.InputTokens > 0 || resp.OutputTokens > 0) {
69			opts.OnUsage(resp.InputTokens, resp.OutputTokens)
70		}
71		if !isEmptyResponse(resp) {
72			return resp, nil
73		}
74
75		log.Warn().
76			Str("provider", opts.Provider.Name()).
77			Int("attempt", attempt+1).
78			Msg("Empty response from provider")
79	}
80
81	return nil, fmt.Errorf("empty response from provider %s", opts.Provider.Name())
82}
83
84func isEmptyResponse(resp *provider.ChatResponse) bool {
85	if resp == nil {
86		return true
87	}
88	return resp.Content == "" && resp.Reasoning == "" && len(resp.ToolCalls) == 0
89}
90
91// emitAssistant builds an assistant message from a ChatResponse, emits it, and appends to history.
92func emitAssistant(opts *ProcessTurnOptions, resp *provider.ChatResponse) {
93	msg := provider.Message{
94		Role:         "assistant",
95		Content:      resp.Content,
96		Reasoning:    resp.Reasoning,
97		ToolCalls:    resp.ToolCalls,
98		CreatedAt:    time.Now(),
99		InputTokens:  resp.InputTokens,
100		OutputTokens: resp.OutputTokens,
101	}
102	if opts.OnMessage != nil {
103		opts.OnMessage(msg)
104	}
105	opts.History = append(opts.History, msg)
106}
107
108// ProcessTurn handles one conversation turn, which may involve tool calls.
109// It streams events via OnDelta and emits complete messages via OnMessage.
110type recentCall struct {
111	Name string
112	Args string
113}
114
115func ProcessTurn(ctx context.Context, opts ProcessTurnOptions) error {
116	if err := normalizeTurnOptions(&opts); err != nil {
117		return err
118	}
119
120	providerTools := toProviderTools(opts.Tools)
121	var recent []recentCall
122	for round := 0; round < opts.MaxToolRounds; round++ {
123		// Inject a <system-reminder> into the last tool result to keep
124		// the model focused. Two sources:
125		// 1. Scratchpad (agent-written plan) — preferred when present.
126		// 2. Goal reminder (user's original request) — fallback.
127		injectRecitation(opts.History, opts.Scratchpad, round)
128
129		resp, err := streamAndCollect(ctx, &opts, providerTools)
130		if err != nil {
131			return fmt.Errorf("LLM stream failed: %w", err)
132		}
133
134		emitAssistant(&opts, resp)
135
136		// If no tool calls, we're done
137		if len(resp.ToolCalls) == 0 {
138			return nil
139		}
140
141		// Notify about tool calls if callback provided
142		if opts.OnToolCall != nil {
143			opts.OnToolCall()
144		}
145
146		// Execute each tool call and update history
147		toolResults := executeToolCalls(ctx, opts.Proxy, resp.ToolCalls, opts.OnMessage)
148		opts.History = append(opts.History, toolResults...)
149		appendRecentCalls(&opts, resp.ToolCalls, toolResults, &recent)
150
151		// Continue loop to let LLM process tool results
152	}
153
154	// Tool call limit reached — do one final call with no tools so the LLM
155	// must reply with text summarizing progress.
156	return finalizeToolLimit(ctx, &opts)
157}
158
159func normalizeTurnOptions(opts *ProcessTurnOptions) error {
160	// Enforce max depth to prevent infinite recursion
161	if opts.Depth > MaxDepth {
162		return fmt.Errorf("max sub-agent depth exceeded: %d > %d", opts.Depth, MaxDepth)
163	}
164	if opts.MaxToolRounds == 0 {
165		opts.MaxToolRounds = 30
166	}
167	return nil
168}
169
170func toProviderTools(tools []mcp.Tool) []provider.Tool {
171	if len(tools) == 0 {
172		return nil
173	}
174	providerTools := make([]provider.Tool, len(tools))
175	for i, t := range tools {
176		providerTools[i] = provider.Tool{
177			Name:        t.Name,
178			Description: t.Description,
179			Parameters:  t.InputSchema,
180		}
181	}
182	return providerTools
183}
184
185func appendRecentCalls(opts *ProcessTurnOptions, toolCalls []provider.ToolCall, toolResults []provider.Message, recent *[]recentCall) {
186	for _, tc := range toolCalls {
187		*recent = append(*recent, recentCall{Name: tc.Name, Args: string(tc.Arguments)})
188	}
189	if len(*recent) < 3 {
190		return
191	}
192	last3 := (*recent)[len(*recent)-3:]
193	if last3[0] == last3[1] && last3[1] == last3[2] {
194		if len(toolResults) > 0 && len(opts.History) > 0 {
195			last := &toolResults[len(toolResults)-1]
196			last.Content += "\n\n<system-reminder>WARNING: You are repeating the same tool call with the same arguments. This is wasteful. Stop and either try a different approach, summarize what you know, or ask the user for help.</system-reminder>"
197			opts.History[len(opts.History)-1] = *last
198		}
199	}
200}
201
202func finalizeToolLimit(ctx context.Context, opts *ProcessTurnOptions) error {
203	if err := ctx.Err(); err != nil {
204		return err
205	}
206	limitMsg := provider.Message{
207		Role:      "user",
208		Content:   "You have exhausted your tool call limit for this turn. Respond in text only. Summarize what you accomplished and what remains.",
209		CreatedAt: time.Now(),
210	}
211	if opts.OnMessage != nil {
212		opts.OnMessage(limitMsg)
213	}
214	opts.History = append(opts.History, limitMsg)
215
216	resp, err := streamAndCollect(ctx, opts, nil)
217	if err != nil {
218		return fmt.Errorf("final text-only LLM stream failed: %w", err)
219	}
220
221	emitAssistant(opts, resp)
222	return nil
223}
224
225// toolCallAccumulator tracks tool calls as they stream in.
226type toolCallAccumulator struct {
227	byIndex     map[int]int
228	calls       []provider.ToolCall
229	argBuilders []string
230}
231
232func newToolCallAccumulator() *toolCallAccumulator {
233	return &toolCallAccumulator{byIndex: make(map[int]int)}
234}
235
236func (a *toolCallAccumulator) begin(evt provider.StreamEvent) {
237	pos := len(a.calls)
238	a.byIndex[evt.ToolCallIndex] = pos
239	a.calls = append(a.calls, provider.ToolCall{ID: evt.ToolCallID, Name: evt.ToolCallName, ThoughtSignature: evt.ToolCallSignature})
240	a.argBuilders = append(a.argBuilders, "")
241}
242
243func (a *toolCallAccumulator) delta(evt provider.StreamEvent) {
244	if pos, ok := a.byIndex[evt.ToolCallIndex]; ok {
245		a.argBuilders[pos] += evt.ToolCallArgs
246	}
247}
248
249func (a *toolCallAccumulator) finalize() []provider.ToolCall {
250	for i := range a.calls {
251		if i < len(a.argBuilders) {
252			a.calls[i].Arguments = json.RawMessage(a.argBuilders[i])
253		}
254	}
255	return a.calls
256}
257
258// collectWithDeltas reads all events from a stream, forwarding each to onDelta,
259// and assembles them into a ChatResponse.
260func collectWithDeltas(ch <-chan provider.StreamEvent, onDelta DeltaCallback) (*provider.ChatResponse, error) {
261	var result provider.ChatResponse
262	tca := newToolCallAccumulator()
263
264	for evt := range ch {
265		if onDelta != nil {
266			onDelta(evt)
267		}
268
269		switch evt.Type {
270		case provider.EventContentDelta:
271			result.Content += evt.Content
272		case provider.EventReasoningDelta:
273			result.Reasoning += evt.Content
274		case provider.EventToolCallBegin:
275			tca.begin(evt)
276		case provider.EventToolCallDelta:
277			tca.delta(evt)
278		case provider.EventUsage:
279			if evt.InputTokens > result.InputTokens {
280				result.InputTokens = evt.InputTokens
281			}
282			if evt.OutputTokens > result.OutputTokens {
283				result.OutputTokens = evt.OutputTokens
284			}
285		case provider.EventError:
286			return nil, evt.Err
287		case provider.EventDone:
288			// finalize
289		}
290	}
291
292	if calls := tca.finalize(); len(calls) > 0 {
293		result.ToolCalls = calls
294	}
295	return &result, nil
296}
297
298// executeToolCalls executes a list of tool calls and adds results to history.
299// Returns the list of tool result messages that were added.
300func executeToolCalls(ctx context.Context, proxy *mcp.Proxy, toolCalls []provider.ToolCall, onMessage MessageCallback) []provider.Message {
301	toolResults := make([]provider.Message, 0, len(toolCalls))
302
303	for _, toolCall := range toolCalls {
304		// Execute tool via MCP proxy
305		result, err := proxy.CallTool(ctx, toolCall.Name, toolCall.Arguments)
306
307		if err != nil {
308			// Add error result to history
309			toolMsg := provider.Message{
310				Role:         "tool",
311				Content:      fmt.Sprintf("Error: %v", err),
312				ToolCallID:   toolCall.ID,
313				FunctionName: toolCall.Name,
314				CreatedAt:    time.Now(),
315			}
316			if onMessage != nil {
317				onMessage(toolMsg)
318			}
319			toolResults = append(toolResults, toolMsg)
320			continue
321		}
322
323		// Check if result is an error
324		if result.IsError {
325			errText := extractTextFromContent(result.Content)
326
327			// Add error result to history
328			toolMsg := provider.Message{
329				Role:         "tool",
330				Content:      errText,
331				ToolCallID:   toolCall.ID,
332				FunctionName: toolCall.Name,
333				CreatedAt:    time.Now(),
334			}
335			if onMessage != nil {
336				onMessage(toolMsg)
337			}
338			toolResults = append(toolResults, toolMsg)
339			continue
340		}
341
342		// Success - extract result text
343		resultText := extractTextFromContent(result.Content)
344
345		// Add tool result to history
346		toolMsg := provider.Message{
347			Role:         "tool",
348			Content:      resultText,
349			ToolCallID:   toolCall.ID,
350			FunctionName: toolCall.Name,
351			CreatedAt:    time.Now(),
352		}
353		if onMessage != nil {
354			onMessage(toolMsg)
355		}
356		toolResults = append(toolResults, toolMsg)
357	}
358
359	return toolResults
360}
361
362// reminderInterval is the number of tool-calling rounds between synthetic
363// goal reminders. After this many rounds the loop injects a system message
364// reciting the user's original request so it stays in the model's recent
365// attention window.
366const reminderInterval = 3
367
368// injectRecitation appends a <system-reminder> block to the last tool-result
369// message in history to keep the model focused during long tool-calling loops.
370// By appending to an existing message instead of creating a new one, we avoid
371// shifting message positions and invalidating the Anthropic prompt cache.
372//
373// Priority: if the agent has written a scratchpad (plan/notes), that is
374// injected. Otherwise the user's original request is echoed as a fallback.
375func injectRecitation(history []provider.Message, pad ScratchpadReader, round int) {
376	if round == 0 || round%reminderInterval != 0 {
377		return
378	}
379
380	// Build the reminder text.
381	var reminder string
382	if pad != nil {
383		if plan := pad.Content(); plan != "" {
384			reminder = plan
385		}
386	}
387	if reminder == "" {
388		// Fallback: echo the user's original request.
389		for _, m := range history {
390			if m.Role == "user" {
391				reminder = "The user's request: " + m.Content
392				break
393			}
394		}
395	}
396	if reminder == "" {
397		return
398	}
399
400	// Append to the last tool-result message, stripping any prior
401	// reminder on that same message to avoid token accumulation.
402	tag := "\n\n<system-reminder>\n"
403	for i := len(history) - 1; i >= 0; i-- {
404		if history[i].Role == "tool" {
405			if idx := strings.Index(history[i].Content, tag); idx >= 0 {
406				history[i].Content = history[i].Content[:idx]
407			}
408			history[i].Content += tag + reminder + "\n</system-reminder>"
409			return
410		}
411	}
412}
413
414// extractTextFromContent extracts text from MCP content blocks.
415func extractTextFromContent(content []mcp.ContentBlock) string {
416	var text string
417	for _, block := range content {
418		if block.Type == "text" {
419			text += block.Text
420		}
421	}
422	return text
423}