62159
Sy
symb / files
symb / internal / llm / loop.go
12.3 KB
1// Package llm implements the LLM interaction loop with tool calling support. 2package llm 3 4import ( 5 "context" 6 "encoding/json" 7 "fmt" 8 "strings" 9 "time" 10 11 "github.com/rs/zerolog/log" 12 "github.com/xonecas/symb/internal/mcp" 13 "github.com/xonecas/symb/internal/provider" 14) 15 16const ( 17 // MaxDepth is the maximum recursion depth for sub-agents. 18 // Matches subagent.MaxSubAgentDepth to prevent import cycle. 19 MaxDepth = 1 20) 21 22// MessageCallback is called when a complete message should be added to history. 23type MessageCallback func(msg provider.Message) 24 25// DeltaCallback is called for each streaming event (content/reasoning deltas). 26type DeltaCallback func(evt provider.StreamEvent) 27 28// ToolCallCallback is called when tool calls are about to be executed. 29type ToolCallCallback func() 30 31// UsageCallback is called with accumulated token usage after each LLM call. 32type UsageCallback func(inputTokens, outputTokens int) 33 34// ScratchpadReader provides read access to the agent's working plan. 35type ScratchpadReader interface { 36 Content() string 37} 38 39// ProcessTurnOptions holds configuration for processing a turn. 40type ProcessTurnOptions struct { 41 Provider provider.Provider 42 Proxy *mcp.Proxy 43 Tools []mcp.Tool 44 History []provider.Message 45 OnMessage MessageCallback 46 OnDelta DeltaCallback // Optional: called for each stream event 47 OnToolCall ToolCallCallback // Optional: called before executing tool calls 48 OnUsage UsageCallback // Optional: called with token usage after each LLM call 49 Scratchpad ScratchpadReader // Optional: agent plan injected at context tail 50 MaxToolRounds int 51 Depth int // Recursion depth (0=root agent, 1=sub-agent) 52} 53 54// streamAndCollect runs one LLM call: streams events, collects the response, 55// reports usage, and returns the ChatResponse. 56func streamAndCollect(ctx context.Context, opts *ProcessTurnOptions, tools []provider.Tool) (*provider.ChatResponse, error) { 57 const maxEmptyRetries = 1 58 59 for attempt := 0; attempt <= maxEmptyRetries; attempt++ { 60 stream, err := opts.Provider.ChatStream(ctx, opts.History, tools) 61 if err != nil { 62 return nil, err 63 } 64 resp, err := collectWithDeltas(stream, opts.OnDelta) 65 if err != nil { 66 return nil, err 67 } 68 if opts.OnUsage != nil && (resp.InputTokens > 0 || resp.OutputTokens > 0) { 69 opts.OnUsage(resp.InputTokens, resp.OutputTokens) 70 } 71 if !isEmptyResponse(resp) { 72 return resp, nil 73 } 74 75 log.Warn(). 76 Str("provider", opts.Provider.Name()). 77 Int("attempt", attempt+1). 78 Msg("Empty response from provider") 79 } 80 81 return nil, fmt.Errorf("empty response from provider %s", opts.Provider.Name()) 82} 83 84func isEmptyResponse(resp *provider.ChatResponse) bool { 85 if resp == nil { 86 return true 87 } 88 return resp.Content == "" && resp.Reasoning == "" && len(resp.ToolCalls) == 0 89} 90 91// emitAssistant builds an assistant message from a ChatResponse, emits it, and appends to history. 92func emitAssistant(opts *ProcessTurnOptions, resp *provider.ChatResponse) { 93 msg := provider.Message{ 94 Role: "assistant", 95 Content: resp.Content, 96 Reasoning: resp.Reasoning, 97 ToolCalls: resp.ToolCalls, 98 CreatedAt: time.Now(), 99 InputTokens: resp.InputTokens, 100 OutputTokens: resp.OutputTokens, 101 } 102 if opts.OnMessage != nil { 103 opts.OnMessage(msg) 104 } 105 opts.History = append(opts.History, msg) 106} 107 108// ProcessTurn handles one conversation turn, which may involve tool calls. 109// It streams events via OnDelta and emits complete messages via OnMessage. 110type recentCall struct { 111 Name string 112 Args string 113} 114 115func ProcessTurn(ctx context.Context, opts ProcessTurnOptions) error { 116 if err := normalizeTurnOptions(&opts); err != nil { 117 return err 118 } 119 120 providerTools := toProviderTools(opts.Tools) 121 var recent []recentCall 122 for round := 0; round < opts.MaxToolRounds; round++ { 123 // Inject a <system-reminder> into the last tool result to keep 124 // the model focused. Two sources: 125 // 1. Scratchpad (agent-written plan) — preferred when present. 126 // 2. Goal reminder (user's original request) — fallback. 127 injectRecitation(opts.History, opts.Scratchpad, round) 128 129 resp, err := streamAndCollect(ctx, &opts, providerTools) 130 if err != nil { 131 return fmt.Errorf("LLM stream failed: %w", err) 132 } 133 134 emitAssistant(&opts, resp) 135 136 // If no tool calls, we're done 137 if len(resp.ToolCalls) == 0 { 138 return nil 139 } 140 141 // Notify about tool calls if callback provided 142 if opts.OnToolCall != nil { 143 opts.OnToolCall() 144 } 145 146 // Execute each tool call and update history 147 toolResults := executeToolCalls(ctx, opts.Proxy, resp.ToolCalls, opts.OnMessage) 148 opts.History = append(opts.History, toolResults...) 149 appendRecentCalls(&opts, resp.ToolCalls, toolResults, &recent) 150 151 // Continue loop to let LLM process tool results 152 } 153 154 // Tool call limit reached — do one final call with no tools so the LLM 155 // must reply with text summarizing progress. 156 return finalizeToolLimit(ctx, &opts) 157} 158 159func normalizeTurnOptions(opts *ProcessTurnOptions) error { 160 // Enforce max depth to prevent infinite recursion 161 if opts.Depth > MaxDepth { 162 return fmt.Errorf("max sub-agent depth exceeded: %d > %d", opts.Depth, MaxDepth) 163 } 164 if opts.MaxToolRounds == 0 { 165 opts.MaxToolRounds = 30 166 } 167 return nil 168} 169 170func toProviderTools(tools []mcp.Tool) []provider.Tool { 171 if len(tools) == 0 { 172 return nil 173 } 174 providerTools := make([]provider.Tool, len(tools)) 175 for i, t := range tools { 176 providerTools[i] = provider.Tool{ 177 Name: t.Name, 178 Description: t.Description, 179 Parameters: t.InputSchema, 180 } 181 } 182 return providerTools 183} 184 185func appendRecentCalls(opts *ProcessTurnOptions, toolCalls []provider.ToolCall, toolResults []provider.Message, recent *[]recentCall) { 186 for _, tc := range toolCalls { 187 *recent = append(*recent, recentCall{Name: tc.Name, Args: string(tc.Arguments)}) 188 } 189 if len(*recent) < 3 { 190 return 191 } 192 last3 := (*recent)[len(*recent)-3:] 193 if last3[0] == last3[1] && last3[1] == last3[2] { 194 if len(toolResults) > 0 && len(opts.History) > 0 { 195 last := &toolResults[len(toolResults)-1] 196 last.Content += "\n\n<system-reminder>WARNING: You are repeating the same tool call with the same arguments. This is wasteful. Stop and either try a different approach, summarize what you know, or ask the user for help.</system-reminder>" 197 opts.History[len(opts.History)-1] = *last 198 } 199 } 200} 201 202func finalizeToolLimit(ctx context.Context, opts *ProcessTurnOptions) error { 203 if err := ctx.Err(); err != nil { 204 return err 205 } 206 limitMsg := provider.Message{ 207 Role: "user", 208 Content: "You have exhausted your tool call limit for this turn. Respond in text only. Summarize what you accomplished and what remains.", 209 CreatedAt: time.Now(), 210 } 211 if opts.OnMessage != nil { 212 opts.OnMessage(limitMsg) 213 } 214 opts.History = append(opts.History, limitMsg) 215 216 resp, err := streamAndCollect(ctx, opts, nil) 217 if err != nil { 218 return fmt.Errorf("final text-only LLM stream failed: %w", err) 219 } 220 221 emitAssistant(opts, resp) 222 return nil 223} 224 225// toolCallAccumulator tracks tool calls as they stream in. 226type toolCallAccumulator struct { 227 byIndex map[int]int 228 calls []provider.ToolCall 229 argBuilders []string 230} 231 232func newToolCallAccumulator() *toolCallAccumulator { 233 return &toolCallAccumulator{byIndex: make(map[int]int)} 234} 235 236func (a *toolCallAccumulator) begin(evt provider.StreamEvent) { 237 pos := len(a.calls) 238 a.byIndex[evt.ToolCallIndex] = pos 239 a.calls = append(a.calls, provider.ToolCall{ID: evt.ToolCallID, Name: evt.ToolCallName, ThoughtSignature: evt.ToolCallSignature}) 240 a.argBuilders = append(a.argBuilders, "") 241} 242 243func (a *toolCallAccumulator) delta(evt provider.StreamEvent) { 244 if pos, ok := a.byIndex[evt.ToolCallIndex]; ok { 245 a.argBuilders[pos] += evt.ToolCallArgs 246 } 247} 248 249func (a *toolCallAccumulator) finalize() []provider.ToolCall { 250 for i := range a.calls { 251 if i < len(a.argBuilders) { 252 a.calls[i].Arguments = json.RawMessage(a.argBuilders[i]) 253 } 254 } 255 return a.calls 256} 257 258// collectWithDeltas reads all events from a stream, forwarding each to onDelta, 259// and assembles them into a ChatResponse. 260func collectWithDeltas(ch <-chan provider.StreamEvent, onDelta DeltaCallback) (*provider.ChatResponse, error) { 261 var result provider.ChatResponse 262 tca := newToolCallAccumulator() 263 264 for evt := range ch { 265 if onDelta != nil { 266 onDelta(evt) 267 } 268 269 switch evt.Type { 270 case provider.EventContentDelta: 271 result.Content += evt.Content 272 case provider.EventReasoningDelta: 273 result.Reasoning += evt.Content 274 case provider.EventToolCallBegin: 275 tca.begin(evt) 276 case provider.EventToolCallDelta: 277 tca.delta(evt) 278 case provider.EventUsage: 279 if evt.InputTokens > result.InputTokens { 280 result.InputTokens = evt.InputTokens 281 } 282 if evt.OutputTokens > result.OutputTokens { 283 result.OutputTokens = evt.OutputTokens 284 } 285 case provider.EventError: 286 return nil, evt.Err 287 case provider.EventDone: 288 // finalize 289 } 290 } 291 292 if calls := tca.finalize(); len(calls) > 0 { 293 result.ToolCalls = calls 294 } 295 return &result, nil 296} 297 298// executeToolCalls executes a list of tool calls and adds results to history. 299// Returns the list of tool result messages that were added. 300func executeToolCalls(ctx context.Context, proxy *mcp.Proxy, toolCalls []provider.ToolCall, onMessage MessageCallback) []provider.Message { 301 toolResults := make([]provider.Message, 0, len(toolCalls)) 302 303 for _, toolCall := range toolCalls { 304 // Execute tool via MCP proxy 305 result, err := proxy.CallTool(ctx, toolCall.Name, toolCall.Arguments) 306 307 if err != nil { 308 // Add error result to history 309 toolMsg := provider.Message{ 310 Role: "tool", 311 Content: fmt.Sprintf("Error: %v", err), 312 ToolCallID: toolCall.ID, 313 FunctionName: toolCall.Name, 314 CreatedAt: time.Now(), 315 } 316 if onMessage != nil { 317 onMessage(toolMsg) 318 } 319 toolResults = append(toolResults, toolMsg) 320 continue 321 } 322 323 // Check if result is an error 324 if result.IsError { 325 errText := extractTextFromContent(result.Content) 326 327 // Add error result to history 328 toolMsg := provider.Message{ 329 Role: "tool", 330 Content: errText, 331 ToolCallID: toolCall.ID, 332 FunctionName: toolCall.Name, 333 CreatedAt: time.Now(), 334 } 335 if onMessage != nil { 336 onMessage(toolMsg) 337 } 338 toolResults = append(toolResults, toolMsg) 339 continue 340 } 341 342 // Success - extract result text 343 resultText := extractTextFromContent(result.Content) 344 345 // Add tool result to history 346 toolMsg := provider.Message{ 347 Role: "tool", 348 Content: resultText, 349 ToolCallID: toolCall.ID, 350 FunctionName: toolCall.Name, 351 CreatedAt: time.Now(), 352 } 353 if onMessage != nil { 354 onMessage(toolMsg) 355 } 356 toolResults = append(toolResults, toolMsg) 357 } 358 359 return toolResults 360} 361 362// reminderInterval is the number of tool-calling rounds between synthetic 363// goal reminders. After this many rounds the loop injects a system message 364// reciting the user's original request so it stays in the model's recent 365// attention window. 366const reminderInterval = 3 367 368// injectRecitation appends a <system-reminder> block to the last tool-result 369// message in history to keep the model focused during long tool-calling loops. 370// By appending to an existing message instead of creating a new one, we avoid 371// shifting message positions and invalidating the Anthropic prompt cache. 372// 373// Priority: if the agent has written a scratchpad (plan/notes), that is 374// injected. Otherwise the user's original request is echoed as a fallback. 375func injectRecitation(history []provider.Message, pad ScratchpadReader, round int) { 376 if round == 0 || round%reminderInterval != 0 { 377 return 378 } 379 380 // Build the reminder text. 381 var reminder string 382 if pad != nil { 383 if plan := pad.Content(); plan != "" { 384 reminder = plan 385 } 386 } 387 if reminder == "" { 388 // Fallback: echo the user's original request. 389 for _, m := range history { 390 if m.Role == "user" { 391 reminder = "The user's request: " + m.Content 392 break 393 } 394 } 395 } 396 if reminder == "" { 397 return 398 } 399 400 // Append to the last tool-result message, stripping any prior 401 // reminder on that same message to avoid token accumulation. 402 tag := "\n\n<system-reminder>\n" 403 for i := len(history) - 1; i >= 0; i-- { 404 if history[i].Role == "tool" { 405 if idx := strings.Index(history[i].Content, tag); idx >= 0 { 406 history[i].Content = history[i].Content[:idx] 407 } 408 history[i].Content += tag + reminder + "\n</system-reminder>" 409 return 410 } 411 } 412} 413 414// extractTextFromContent extracts text from MCP content blocks. 415func extractTextFromContent(content []mcp.ContentBlock) string { 416 var text string 417 for _, block := range content { 418 if block.Type == "text" { 419 text += block.Text 420 } 421 } 422 return text 423}