xonecas.io
60255
Zn

Zoea-Nova / commit 57ced1b

fix(core,provider): align mysis states and tool retries

xonecas2026-02-07 57ced1b5b3da9386501d267e9029b9bbe5526876 parent 1c1a104

10 files changed

This diff is too big to show in full. The first 1000 lines are below, and every file is listed with its counts.

documentation/architecture/MYSIS_STATE_MACHINE.md+17−6
@@ -4,10 +4,15 @@ This document describes the valid Mysis lifecycle states and the transitions bet
44
55 ## States
66
7-- `idle`: Created or loaded but not running.
8-- `running`: Processing messages and tool calls.
7+- `idle`: Not running. Set on creation and when a mysis fails 3 nudges; requires a commander message/broadcast to resume.
8+- `running`: Active and eligible for nudges; waiting between loop iterations.
99 - `stopped`: Explicitly stopped by user action.
10-- `errored`: Entered due to an error during execution or startup.
10+- `errored`: Provider or MCP failures after retries; recorded as `lastError`.
11+
12+## Activity (not a state)
13+
14+- `thinking` is represented by `ActivityStateLLMCall` (not a MysisState).
15+- In-game activities (traveling, cooldown, etc.) are for context/TUI only and do not block nudges.
1116
1217 ## Diagram
1318
@@ -15,6 +20,7 @@ This document describes the valid Mysis lifecycle states and the transitions bet
1520 stateDiagram-v2
1621 [*] --> idle: create or load
1722 idle --> running: start
23+ running --> idle: 3 nudges failed
1824 running --> stopped: stop
1925 running --> errored: error
2026 stopped --> running: relaunch
@@ -32,7 +38,7 @@ stateDiagram-v2
3238
3339 - Triggered by `Commander.StartMysis` (TUI `r` key).
3440 - Transitions to `running` from `idle`, `stopped`, or `errored`.
35-- If the store update fails during start, the Mysis enters `errored`.
41+- If the store update fails during start, the Mysis remains unchanged and returns an error.
3642
3743 ### Stop
3844
@@ -42,8 +48,13 @@ stateDiagram-v2
4248
4349 ### Error
4450
45-- Triggered by `setErrorState` on execution failures.
46-- Transitions to `errored` and records `lastError`.
51+ - Triggered by `setErrorState` on execution failures (provider or MCP after retries).
52+ - Transitions to `errored` and records `lastError`.
53+
54+### Idle (Nudge Breaker)
55+
56+- Triggered after 3 failed nudges with no progress.
57+- Transitions to `idle` and signals that manual attention is needed.
4758
4859 ## Notes
4960
documentation/plans/2026-02-07-mysis-state-alignment.md+245−0
@@ -0,0 +1,245 @@
1+# Mysis State Alignment Implementation Plan
2+
3+> **For Claude:** REQUIRED SUB-SKILL: Use superpowers:executing-plans to implement this plan task-by-task.
4+
5+**Goal:** Align mysis state semantics with the desired behavior (idle on 3 nudges, errored on provider/MCP failures, activities only for TUI/context).
6+
7+**Architecture:** Keep `MysisState` for lifecycle (idle/running/stopped/errored) and use `ActivityState` only for UI/context. Adjust loop control to avoid activity gating, add explicit idle transition for the 3‑nudge breaker, and treat tool timeouts/failures as errored with persisted error context.
8+
9+**Tech Stack:** Go 1.22, Bubble Tea, SQLite store, zerolog
10+
11+---
12+
13+## Task 1: Update state semantics documentation
14+
15+**Files:**
16+- Modify: `documentation/architecture/MYSIS_STATE_MACHINE.md`
17+
18+**Step 1: Write the doc change**
19+
20+- Update definitions:
21+ - `idle`: set when 3 nudges fail; requires commander/broadcast to resume.
22+ - `thinking`: not a state; represent via `ActivityStateLLMCall`.
23+ - `running`: after reply, waiting for next nudge.
24+ - `errored`: provider/MCP failures after retries.
25+- Update transitions to include `running → idle` on 3‑nudge failure.
26+
27+**Step 2: Review for consistency**
28+
29+- Ensure doc does not claim activity gating for nudges.
30+
31+**Step 3: Commit**
32+
33+```bash
34+git add documentation/architecture/MYSIS_STATE_MACHINE.md
35+git commit -m "docs: align mysis state semantics"
36+```
37+
38+---
39+
40+## Task 2: Make activity non‑blocking for nudges
41+
42+**Files:**
43+- Modify: `internal/core/mysis.go` (`shouldNudge`)
44+
45+**Step 1: Write failing unit test**
46+
47+- Add test in `internal/core/mysis_test.go`:
48+ - Set activity to `Traveling` or `Cooldown` with `activityUntil` in the future.
49+ - Assert `shouldNudge(...)` returns `true`.
50+
51+**Step 2: Run test to verify it fails**
52+
53+Run:
54+```bash
55+go test ./internal/core -run TestShouldNudge_IgnoresActivity
56+```
57+Expected: FAIL (returns false today).
58+
59+**Step 3: Minimal implementation**
60+
61+- Change `shouldNudge` to ignore activity state and always return true when running.
62+- Keep activity state for UI/context only.
63+
64+**Step 4: Run test to verify it passes**
65+
66+Run:
67+```bash
68+go test ./internal/core -run TestShouldNudge_IgnoresActivity
69+```
70+Expected: PASS.
71+
72+**Step 5: Commit**
73+
74+```bash
75+git add internal/core/mysis.go internal/core/mysis_test.go
76+git commit -m "fix: keep nudges active during in-game activity"
77+```
78+
79+---
80+
81+## Task 3: 3‑nudge breaker sets Idle (not Errored)
82+
83+**Files:**
84+- Modify: `internal/core/mysis.go` (`run`, add `setIdle` helper)
85+- Modify: `internal/store/myses.go` if needed (state update helper)
86+
87+**Step 1: Write failing unit test**
88+
89+- Add test in `internal/core/mysis_test.go`:
90+ - Simulate 3 idle nudges with no turn in progress.
91+ - Assert state transitions to `idle` and store updated.
92+
93+**Step 2: Run test to verify it fails**
94+
95+Run:
96+```bash
97+go test ./internal/core -run TestNudgeBreakerSetsIdle
98+```
99+Expected: FAIL (currently sets errored).
100+
101+**Step 3: Minimal implementation**
102+
103+- Add `setIdle(reason string)` that:
104+ - Sets `state = idle`
105+ - Updates store
106+ - Emits `EventMysisStateChanged`
107+ - Publishes a non‑error event or log with reason
108+- Use `setIdle("Failed to respond after 3 nudges")` in `run` instead of `setError`.
109+
110+**Step 4: Run test to verify it passes**
111+
112+Run:
113+```bash
114+go test ./internal/core -run TestNudgeBreakerSetsIdle
115+```
116+Expected: PASS.
117+
118+**Step 5: Commit**
119+
120+```bash
121+git add internal/core/mysis.go internal/core/mysis_test.go
122+git commit -m "fix: mark mysis idle after nudge breaker"
123+```
124+
125+---
126+
127+## Task 4: Tool timeout/failure → errored + persisted tool error message
128+
129+**Files:**
130+- Modify: `internal/core/mysis.go` (`SendMessageFrom`, `SendEphemeralMessage`, `executeToolCall`)
131+- Modify: `internal/constants/constants.go` (add error message template)
132+
133+**Step 1: Write failing unit test**
134+
135+- Add test in `internal/core/mysis_test.go`:
136+ - Simulate a tool call that times out (context deadline exceeded).
137+ - Assert:
138+ - `MysisState` becomes `errored`.
139+ - A tool result memory is stored that contains a clear timeout error string.
140+
141+**Step 2: Run test to verify it fails**
142+
143+Run:
144+```bash
145+go test ./internal/core -run TestToolTimeoutSetsErroredAndPersistsToolError
146+```
147+Expected: FAIL.
148+
149+**Step 3: Minimal implementation**
150+
151+- In tool execution loop:
152+ - If `executeToolCall` returns `ctx.DeadlineExceeded` or `ctx.Canceled`, store a tool result memory with `Error: tool call <name> timed out: <err>`.
153+ - Call `setError(err)` immediately after persisting the tool error memory.
154+- Ensure the error text is included in stored memory so a restarted mysis can read it.
155+
156+**Step 4: Run test to verify it passes**
157+
158+Run:
159+```bash
160+go test ./internal/core -run TestToolTimeoutSetsErroredAndPersistsToolError
161+```
162+Expected: PASS.
163+
164+**Step 5: Commit**
165+
166+```bash
167+git add internal/core/mysis.go internal/core/mysis_test.go internal/constants/constants.go
168+git commit -m "fix: mark mysis errored on tool timeout and persist error"
169+```
170+
171+---
172+
173+## Task 5: MCP failure after retries → errored
174+
175+**Files:**
176+- Modify: `internal/core/mysis.go` (tool call error handling)
177+- Modify: `internal/mcp/proxy.go` if needed for retry exhaust signal
178+
179+**Step 1: Write failing unit test**
180+
181+- Add test in `internal/core/mysis_test.go`:
182+ - Simulate MCP call returning a retried failure (tool error) without store errors.
183+ - Assert mysis transitions to `errored` and tool error message persisted.
184+
185+**Step 2: Run test to verify it fails**
186+
187+Run:
188+```bash
189+go test ./internal/core -run TestToolRetryFailureSetsErrored
190+```
191+Expected: FAIL.
192+
193+**Step 3: Minimal implementation**
194+
195+- Detect retry‑exhausted tool failure (add a sentinel error or wrapper in MCP proxy).
196+- When detected, store error tool result and call `setError(err)`.
197+
198+**Step 4: Run test to verify it passes**
199+
200+Run:
201+```bash
202+go test ./internal/core -run TestToolRetryFailureSetsErrored
203+```
204+Expected: PASS.
205+
206+**Step 5: Commit**
207+
208+```bash
209+git add internal/core/mysis.go internal/mcp/proxy.go internal/core/mysis_test.go
210+git commit -m "fix: set errored on MCP retry exhaustion"
211+```
212+
213+---
214+
215+## Task 6: Verify behavior end‑to‑end
216+
217+**Step 1: Run unit tests**
218+
219+Run:
220+```bash
221+go test ./internal/core
222+```
223+
224+**Step 2: Run broader test suite**
225+
226+Run:
227+```bash
228+make test
229+```
230+
231+**Step 3: Build**
232+
233+Run:
234+```bash
235+make build
236+```
237+
238+---
239+
240+## Notes / Decisions
241+
242+- `Thinking` remains an activity indicator (`ActivityStateLLMCall`), not a state.
243+- Activities remain for UI/context and no longer block nudges.
244+- `Idle` is now the “needs attention” state after 3 failed nudges.
245+- Provider errors already set `Errored`; MCP failures will now do so when retries are exhausted.
internal/core/activity_test.go+22−6
@@ -21,7 +21,7 @@ func TestMysis_ShouldNudge_Idle(t *testing.T) {
2121 }
2222 }
2323
24-// TestMysis_ShouldNudge_Traveling_InFuture tests nudge suppression during travel.
24+// TestMysis_ShouldNudge_Traveling_InFuture tests nudges during travel.
2525 func TestMysis_ShouldNudge_Traveling_InFuture(t *testing.T) {
2626 m := &Mysis{
2727 activityState: ActivityStateTraveling,
@@ -30,8 +30,16 @@ func TestMysis_ShouldNudge_Traveling_InFuture(t *testing.T) {
3030
3131 now := time.Now()
3232
33- if m.shouldNudge(now) {
34- t.Error("Expected no nudge while traveling (activity until time in future)")
33+ if !m.shouldNudge(now) {
34+ t.Error("Expected nudge while traveling (activity until time in future)")
35+ }
36+
37+ if m.activityState != ActivityStateTraveling {
38+ t.Errorf("Expected activity state traveling to remain, got %s", m.activityState)
39+ }
40+
41+ if m.activityUntil.IsZero() {
42+ t.Error("Expected activityUntil to remain set")
3543 }
3644 }
3745
@@ -58,7 +66,7 @@ func TestMysis_ShouldNudge_Traveling_Arrived(t *testing.T) {
5866 }
5967 }
6068
61-// TestMysis_ShouldNudge_Cooldown_Active tests nudge suppression during cooldown.
69+// TestMysis_ShouldNudge_Cooldown_Active tests nudges during cooldown.
6270 func TestMysis_ShouldNudge_Cooldown_Active(t *testing.T) {
6371 m := &Mysis{
6472 activityState: ActivityStateCooldown,
@@ -67,8 +75,16 @@ func TestMysis_ShouldNudge_Cooldown_Active(t *testing.T) {
6775
6876 now := time.Now()
6977
70- if m.shouldNudge(now) {
71- t.Error("Expected no nudge during active cooldown")
78+ if !m.shouldNudge(now) {
79+ t.Error("Expected nudge during active cooldown")
80+ }
81+
82+ if m.activityState != ActivityStateCooldown {
83+ t.Errorf("Expected activity state cooldown to remain, got %s", m.activityState)
84+ }
85+
86+ if m.activityUntil.IsZero() {
87+ t.Error("Expected activityUntil to remain set")
7288 }
7389 }
7490
internal/core/mysis.go+79−11
@@ -581,6 +581,18 @@ func (m *Mysis) SendMessageFrom(content string, source store.MemorySource, sende
581581 Message: &MessageData{Role: "tool", Content: fmt.Sprintf("[%s] %s", tc.Name, a.formatToolResultDisplay(result, execErr))},
582582 Timestamp: time.Now(),
583583 })
584+
585+ if execErr != nil && isToolTimeout(execErr) {
586+ a.bus.Publish(Event{Type: EventNetworkIdle, MysisID: a.id, Timestamp: time.Now()})
587+ a.setError(execErr)
588+ return fmt.Errorf("tool call timed out: %w", execErr)
589+ }
590+
591+ if execErr != nil && isToolRetryExhausted(execErr) {
592+ a.bus.Publish(Event{Type: EventNetworkIdle, MysisID: a.id, Timestamp: time.Now()})
593+ a.setError(execErr)
594+ return fmt.Errorf("tool call failed after retries: %w", execErr)
595+ }
584596 }
585597
586598 // Continue loop to get next LLM response
@@ -858,6 +870,18 @@ func (m *Mysis) SendEphemeralMessage(content string, source store.MemorySource)
858870 Message: &MessageData{Role: "tool", Content: fmt.Sprintf("[%s] %s", tc.Name, a.formatToolResultDisplay(result, execErr))},
859871 Timestamp: time.Now(),
860872 })
873+
874+ if execErr != nil && isToolTimeout(execErr) {
875+ a.bus.Publish(Event{Type: EventNetworkIdle, MysisID: a.id, Timestamp: time.Now()})
876+ a.setError(execErr)
877+ return fmt.Errorf("tool call timed out: %w", execErr)
878+ }
879+
880+ if execErr != nil && isToolRetryExhausted(execErr) {
881+ a.bus.Publish(Event{Type: EventNetworkIdle, MysisID: a.id, Timestamp: time.Now()})
882+ a.setError(execErr)
883+ return fmt.Errorf("tool call failed after retries: %w", execErr)
884+ }
861885 }
862886
863887 // Continue loop to get next LLM response
@@ -1097,7 +1121,13 @@ func (m *Mysis) parseStoredToolCalls(stored string) []provider.ToolCall {
10971121 // formatToolResult formats a tool result for storage (includes ID for LLM context).
10981122 func (m *Mysis) formatToolResult(toolCallID, toolName string, result *mcp.ToolResult, err error) string {
10991123 if err != nil {
1100- return fmt.Sprintf("%s:Error calling %s: %v. Check the tool's required parameters and try again.", toolCallID, toolName, err)
1124+ if errors.Is(err, context.DeadlineExceeded) {
1125+ return fmt.Sprintf("%s%sError calling %s: tool call timed out: %v", toolCallID, constants.ToolCallStorageFieldDelimiter, toolName, err)
1126+ }
1127+ if errors.Is(err, mcp.ErrToolRetryExhausted) {
1128+ return fmt.Sprintf("%s%sError calling %s: MCP call failed after retries: %v", toolCallID, constants.ToolCallStorageFieldDelimiter, toolName, err)
1129+ }
1130+ return fmt.Sprintf("%s%sError calling %s: %v. Check the tool's required parameters and try again.", toolCallID, constants.ToolCallStorageFieldDelimiter, toolName, err)
11011131 }
11021132
11031133 var texts []string
@@ -1119,6 +1149,14 @@ func (m *Mysis) formatToolResult(toolCallID, toolName string, result *mcp.ToolRe
11191149 return fmt.Sprintf("%s%s%s", toolCallID, constants.ToolCallStorageFieldDelimiter, content)
11201150 }
11211151
1152+func isToolTimeout(err error) bool {
1153+ return errors.Is(err, context.DeadlineExceeded)
1154+}
1155+
1156+func isToolRetryExhausted(err error) bool {
1157+ return errors.Is(err, mcp.ErrToolRetryExhausted)
1158+}
1159+
11221160 // formatToolResultDisplay formats a tool result for UI display (human-readable).
11231161 func (m *Mysis) formatToolResultDisplay(result *mcp.ToolResult, err error) string {
11241162 if err != nil {
@@ -1192,6 +1230,40 @@ func (m *Mysis) setError(err error) {
11921230 })
11931231 }
11941232
1233+func (m *Mysis) setIdle(reason string) {
1234+ a := m
1235+ a.mu.Lock()
1236+ oldState := a.state
1237+
1238+ if oldState == MysisStateStopped {
1239+ a.mu.Unlock()
1240+ log.Debug().Str("mysis", a.name).Str("reason", reason).Msg("Ignoring idle transition - mysis was intentionally stopped")
1241+ return
1242+ }
1243+
1244+ if oldState == MysisStateIdle {
1245+ a.mu.Unlock()
1246+ return
1247+ }
1248+
1249+ log.Warn().
1250+ Str("mysis", a.name).
1251+ Str("old_state", string(oldState)).
1252+ Str("reason", reason).
1253+ Msg("Mysis transitioning to idle state")
1254+
1255+ a.state = MysisStateIdle
1256+ a.lastError = nil
1257+ a.mu.Unlock()
1258+
1259+ if updateErr := a.store.UpdateMysisState(a.id, store.MysisStateIdle); updateErr != nil {
1260+ log.Warn().Err(updateErr).Str("mysis", a.id).Msg("Failed to update state in store")
1261+ }
1262+
1263+ a.emitStateChange(oldState, MysisStateIdle)
1264+ a.releaseCurrentAccount()
1265+}
1266+
11951267 // getContextMemories returns memories for LLM context: system prompt + recent messages.
11961268 // This keeps context small for faster inference while preserving essential information.
11971269 // Compacts repeated snapshot tool results to prefer recent state.
@@ -1449,7 +1521,7 @@ func (m *Mysis) run(ctx context.Context) {
14491521
14501522 // If we've failed 3 times, transition to error state
14511523 if failCount >= 3 {
1452- a.setError(errors.New("Failed to respond after 3 nudges"))
1524+ a.setIdle("Failed to respond after 3 nudges")
14531525 return
14541526 }
14551527
@@ -1598,16 +1670,12 @@ func (m *Mysis) shouldNudge(now time.Time) bool {
15981670 a.mu.Lock()
15991671 defer a.mu.Unlock()
16001672
1601- if a.activityState != ActivityStateTraveling && a.activityState != ActivityStateCooldown {
1602- return true
1603- }
1604-
1605- if !a.activityUntil.IsZero() && now.Before(a.activityUntil) {
1606- return false
1673+ if a.activityState == ActivityStateTraveling || a.activityState == ActivityStateCooldown {
1674+ if !a.activityUntil.IsZero() && !now.Before(a.activityUntil) {
1675+ a.activityState = ActivityStateIdle
1676+ a.activityUntil = time.Time{}
1677+ }
16071678 }
1608-
1609- a.activityState = ActivityStateIdle
1610- a.activityUntil = time.Time{}
16111679 return true
16121680 }
16131681
internal/core/mysis_tools_test.go+159−0
@@ -13,6 +13,23 @@ import (
1313 "github.com/xonecas/zoea-nova/internal/store"
1414 )
1515
16+type failingUpstream struct {
17+ tools []mcp.Tool
18+ callErr error
19+}
20+
21+func (f *failingUpstream) Initialize(ctx context.Context, clientInfo map[string]interface{}) (*mcp.Response, error) {
22+ return nil, nil
23+}
24+
25+func (f *failingUpstream) ListTools(ctx context.Context) ([]mcp.Tool, error) {
26+ return f.tools, nil
27+}
28+
29+func (f *failingUpstream) CallTool(ctx context.Context, name string, arguments interface{}) (*mcp.ToolResult, error) {
30+ return nil, f.callErr
31+}
32+
1633 func TestMysisToolExecution(t *testing.T) {
1734 s, bus, cleanup := setupMysisTest(t)
1835 defer cleanup()
@@ -195,3 +212,145 @@ func TestMysisToolError(t *testing.T) {
195212 t.Error("tool error not found in memories")
196213 }
197214 }
215+
216+func TestMysisToolTimeoutSetsErrored(t *testing.T) {
217+ s, bus, cleanup := setupMysisTest(t)
218+ defer cleanup()
219+
220+ stored, _ := s.CreateMysis("tool-timeout-mysis", "mock", "test-model", 0.7)
221+
222+ mock := provider.NewMock("mock", "Initial response")
223+ mysis := NewMysis(stored.ID, stored.Name, stored.CreatedAt, mock, s, bus)
224+
225+ proxy := mcp.NewProxy(nil)
226+ proxy.RegisterTool(mcp.Tool{
227+ Name: "timeout_tool",
228+ }, func(ctx context.Context, args json.RawMessage) (*mcp.ToolResult, error) {
229+ return nil, context.DeadlineExceeded
230+ })
231+ mysis.SetMCP(proxy)
232+
233+ // Start and wait for initial turn
234+ events := bus.Subscribe()
235+ mysis.Start()
236+
237+ timeout := time.After(5 * time.Second)
238+ initialFinished := false
239+ for !initialFinished {
240+ select {
241+ case e := <-events:
242+ if e.Type == EventMysisResponse {
243+ initialFinished = true
244+ }
245+ case <-timeout:
246+ t.Fatal("timeout waiting for initial turn")
247+ }
248+ }
249+
250+ // Trigger tool call that times out
251+ mock.WithToolCalls([]provider.ToolCall{
252+ {
253+ ID: "call_timeout",
254+ Name: "timeout_tool",
255+ Arguments: json.RawMessage(`{}`),
256+ },
257+ })
258+
259+ go mysis.SendMessage("Trigger timeout", store.MemorySourceDirect)
260+
261+ deadline := time.Now().Add(2 * time.Second)
262+ for time.Now().Before(deadline) {
263+ if mysis.State() == MysisStateErrored {
264+ break
265+ }
266+ time.Sleep(50 * time.Millisecond)
267+ }
268+
269+ if state := mysis.State(); state != MysisStateErrored {
270+ t.Fatalf("expected state=errored, got %s", state)
271+ }
272+
273+ // Verify memory contains timeout error
274+ foundTimeout := false
275+ memories, _ := s.GetMemories(stored.ID)
276+ for _, m := range memories {
277+ if m.Role == store.MemoryRoleTool && strings.Contains(m.Content, "call_timeout") && strings.Contains(m.Content, "timed out") {
278+ foundTimeout = true
279+ break
280+ }
281+ }
282+
283+ if !foundTimeout {
284+ t.Error("tool timeout error not found in memories")
285+ }
286+}
287+
288+func TestMysisToolRetryExhaustionSetsErrored(t *testing.T) {
289+ s, bus, cleanup := setupMysisTest(t)
290+ defer cleanup()
291+
292+ stored, _ := s.CreateMysis("tool-retry-mysis", "mock", "test-model", 0.7)
293+
294+ mock := provider.NewMock("mock", "Initial response")
295+ mysis := NewMysis(stored.ID, stored.Name, stored.CreatedAt, mock, s, bus)
296+
297+ upstream := &failingUpstream{
298+ tools: []mcp.Tool{{Name: "upstream_tool"}},
299+ callErr: errors.New("upstream unavailable"),
300+ }
301+ mysis.SetMCP(mcp.NewProxy(upstream))
302+
303+ // Start and wait for initial turn
304+ events := bus.Subscribe()
305+ mysis.Start()
306+
307+ timeout := time.After(5 * time.Second)
308+ initialFinished := false
309+ for !initialFinished {
310+ select {
311+ case e := <-events:
312+ if e.Type == EventMysisResponse {
313+ initialFinished = true
314+ }
315+ case <-timeout:
316+ t.Fatal("timeout waiting for initial turn")
317+ }
318+ }
319+
320+ // Trigger upstream tool call
321+ mock.WithToolCalls([]provider.ToolCall{
322+ {
323+ ID: "call_retry",
324+ Name: "upstream_tool",
325+ Arguments: json.RawMessage(`{}`),
326+ },
327+ })
328+
329+ go mysis.SendMessage("Trigger upstream retry", store.MemorySourceDirect)
330+
331+ deadline := time.Now().Add(4 * time.Second)
332+ for time.Now().Before(deadline) {
333+ if mysis.State() == MysisStateErrored {
334+ break
335+ }
336+ time.Sleep(50 * time.Millisecond)
337+ }
338+
339+ if state := mysis.State(); state != MysisStateErrored {
340+ t.Fatalf("expected state=errored, got %s", state)
341+ }
342+
343+ // Verify memory contains retry exhaustion error
344+ foundRetry := false
345+ memories, _ := s.GetMemories(stored.ID)
346+ for _, m := range memories {
347+ if m.Role == store.MemoryRoleTool && strings.Contains(m.Content, "call_retry") && strings.Contains(m.Content, "failed after retries") {
348+ foundRetry = true
349+ break
350+ }
351+ }
352+
353+ if !foundRetry {
354+ t.Error("tool retry exhaustion error not found in memories")
355+ }
356+}
internal/core/state_machine_test.go+39−0
@@ -253,6 +253,45 @@ func TestStateTransition_Running_To_Errored(t *testing.T) {
253253 }
254254 }
255255
256+// TestStateTransition_Running_To_Idle tests the running → idle transition.
257+// Trigger: nudge breaker (simulated)
258+// Expected: State transitions to Idle, LastError is nil
259+func TestStateTransition_Running_To_Idle(t *testing.T) {
260+ cmd, cleanup := setupStateMachineTest(t)
261+ defer cleanup()
262+
263+ // Create and start mysis
264+ mysis, err := cmd.CreateMysis("test-running-idle", "mock")
265+ if err != nil {
266+ t.Fatalf("CreateMysis() error: %v", err)
267+ }
268+
269+ if err := mysis.Start(); err != nil {
270+ t.Fatalf("Start() error: %v", err)
271+ }
272+
273+ // Simulate nudge breaker
274+ mysis.setIdle("Failed to respond after 3 nudges")
275+
276+ // Verify idle state
277+ if state := mysis.State(); state != MysisStateIdle {
278+ t.Errorf("expected state=idle, got %s", state)
279+ }
280+
281+ if lastErr := mysis.LastError(); lastErr != nil {
282+ t.Errorf("expected LastError=nil after idle transition, got %v", lastErr)
283+ }
284+
285+ // Verify store persistence
286+ stored, err := cmd.Store().GetMysis(mysis.ID())
287+ if err != nil {
288+ t.Fatalf("GetMysis() error: %v", err)
289+ }
290+ if stored.State != store.MysisStateIdle {
291+ t.Errorf("expected stored state=idle, got %s", stored.State)
292+ }
293+}
294+
256295 // TestStateTransition_Stopped_To_Running tests the stopped → running transition.
257296 // Trigger: Start() (relaunch)
258297 // Expected: State transitions to Running, LastError cleared
internal/mcp/proxy.go+42−1
@@ -3,9 +3,11 @@ package mcp
33 import (
44 "context"
55 "encoding/json"
6+ "errors"
67 "fmt"
78 "strings"
89 "sync"
10+ "time"
911
1012 "github.com/rs/zerolog/log"
1113 )
@@ -44,6 +46,12 @@ type Proxy struct {
4446 accountStore AccountStore
4547 }
4648
49+var (
50+ ErrToolRetryExhausted = errors.New("mcp tool call failed after retries")
51+)
52+
53+var toolRetryDelays = []time.Duration{200 * time.Millisecond, 400 * time.Millisecond, 800 * time.Millisecond}
54+
4755 // NewProxy creates a new MCP proxy.
4856 func NewProxy(upstream UpstreamClient) *Proxy {
4957 return &Proxy{
@@ -130,7 +138,7 @@ func (p *Proxy) CallTool(ctx context.Context, caller CallerContext, name string,
130138 }
131139 }
132140
133- result, err := p.upstream.CallTool(ctx, name, args)
141+ result, err := p.callUpstreamWithRetry(ctx, name, args)
134142
135143 if accountStore != nil && result != nil && !result.IsError {
136144 p.interceptAuthTools(name, arguments, result)
@@ -145,6 +153,39 @@ func (p *Proxy) CallTool(ctx context.Context, caller CallerContext, name string,
145153 }, nil
146154 }
147155
156+func (p *Proxy) callUpstreamWithRetry(ctx context.Context, name string, args interface{}) (*ToolResult, error) {
157+ var lastErr error
158+ for attempt := 0; attempt <= len(toolRetryDelays); attempt++ {
159+ if attempt > 0 {
160+ delay := toolRetryDelays[attempt-1]
161+ log.Warn().
162+ Str("tool", name).
163+ Int("attempt", attempt).
164+ Dur("delay", delay).
165+ Msg("Retrying MCP tool call after error")
166+
167+ select {
168+ case <-time.After(delay):
169+ case <-ctx.Done():
170+ return nil, ctx.Err()
171+ }
172+ }
173+
174+ result, err := p.upstream.CallTool(ctx, name, args)
175+ if err == nil {
176+ return result, nil
177+ }
178+
179+ if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
180+ return nil, err
181+ }
182+
183+ lastErr = err
184+ }
185+
186+ return nil, fmt.Errorf("%w: %v", ErrToolRetryExhausted, lastErr)
187+}
188+
148189 func (p *Proxy) interceptAuthTools(toolName string, arguments json.RawMessage, result *ToolResult) {
149190 switch toolName {
150191 case "register":
internal/provider/factory.go+2−1
@@ -27,10 +27,11 @@ type OpenCodeFactory struct {
2727 }
2828
2929 func NewOpenCodeFactory(endpoint, apiKey string, rateLimit float64, rateBurst int) *OpenCodeFactory {
30+ burst := 1
3031 return &OpenCodeFactory{
3132 endpoint: endpoint,
3233 apiKey: apiKey,
33- limiter: rate.NewLimiter(rate.Limit(rateLimit), rateBurst),
34+ limiter: rate.NewLimiter(rate.Limit(rateLimit), burst),
3435 }
3536 }
3637
internal/provider/ollama.go+76−30
@@ -9,6 +9,7 @@ import (
99 "io"
1010 "net/http"
1111 "strings"
12+ "time"
1213
1314 "github.com/rs/zerolog/log"
1415 openai "github.com/sashabaranov/go-openai"
@@ -25,6 +26,8 @@ type OllamaProvider struct {
2526 limiter *rate.Limiter
2627 }
2728
29+var ollamaRetryDelays = []time.Duration{5 * time.Second, 10 * time.Second, 15 * time.Second}
30+
2831 // NewOllama creates a new Ollama provider.
2932 // Ollama exposes an OpenAI-compatible API at /v1.
3033 func NewOllama(endpoint, model string) *OllamaProvider {
@@ -53,12 +56,6 @@ func (p *OllamaProvider) Name() string {
5356
5457 // Chat sends messages and returns the complete response.
5558 func (p *OllamaProvider) Chat(ctx context.Context, messages []Message) (string, error) {
56- if p.limiter != nil {
57- if err := p.limiter.Wait(ctx); err != nil {
58- return "", err
59- }
60- }
61-
6259 resp, err := p.createChatCompletion(ctx, ollamaChatRequest{
6360 Model: p.model,
6461 Messages: mergeConsecutiveSystemMessagesOllama(toOllamaMessages(messages)),
@@ -77,12 +74,6 @@ func (p *OllamaProvider) Chat(ctx context.Context, messages []Message) (string,
7774
7875 // ChatWithTools sends messages with available tools and returns response with potential tool calls.
7976 func (p *OllamaProvider) ChatWithTools(ctx context.Context, messages []Message, tools []Tool) (*ChatResponse, error) {
80- if p.limiter != nil {
81- if err := p.limiter.Wait(ctx); err != nil {
82- return nil, err
83- }
84- }
85-
8677 resp, err := p.createChatCompletion(ctx, ollamaChatRequest{
8778 Model: p.model,
8879 Messages: mergeConsecutiveSystemMessagesOllama(toOllamaMessages(messages)),
@@ -203,29 +194,84 @@ func (p *OllamaProvider) createChatCompletion(ctx context.Context, req ollamaCha
203194 }
204195
205196 url := p.baseURL + "/chat/completions"
206- httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
207- if err != nil {
208- return nil, err
209- }
210- httpReq.Header.Set("Content-Type", "application/json")
211197
212- resp, err := p.httpClient.Do(httpReq)
213- if err != nil {
214- return nil, err
215- }
216- defer resp.Body.Close()
198+ maxRetries := len(ollamaRetryDelays)
199+ var lastErr error
200+ for attempt := 0; attempt <= maxRetries; attempt++ {
201+ if attempt > 0 {
202+ delay := ollamaRetryDelays[attempt-1]
203+ log.Warn().
204+ Str("provider", "ollama").
205+ Int("attempt", attempt).
206+ Dur("delay", delay).
207+ Msg("Retrying Ollama request after transient error")
208+
209+ select {
210+ case <-time.After(delay):
211+ case <-ctx.Done():
212+ return nil, ctx.Err()
213+ }
214+ }
217215
218- if resp.StatusCode < 200 || resp.StatusCode >= 300 {
219- payload, _ := io.ReadAll(resp.Body)
220- return nil, fmt.Errorf("chat completion status %d: %s", resp.StatusCode, strings.TrimSpace(string(payload)))
221- }
216+ if p.limiter != nil {
217+ if err := p.limiter.Wait(ctx); err != nil {
218+ return nil, err
219+ }
220+ }
222221
223- var decoded chatCompletionResponse
224- if err := json.NewDecoder(resp.Body).Decode(&decoded); err != nil {
225- return nil, err
222+ httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(body))
223+ if err != nil {
224+ return nil, err
225+ }
226+ httpReq.Header.Set("Content-Type", "application/json")
227+
228+ resp, err := p.httpClient.Do(httpReq)
229+ if err != nil {
230+ lastErr = err
231+ continue
232+ }
233+
234+ if resp.StatusCode == 429 || resp.StatusCode == 500 || resp.StatusCode == 502 ||
235+ resp.StatusCode == 503 || resp.StatusCode == 504 {
236+ payload, _ := io.ReadAll(resp.Body)
237+ resp.Body.Close()
238+ lastErr = fmt.Errorf("chat completion status %d: %s", resp.StatusCode, strings.TrimSpace(string(payload)))
239+
240+ log.Warn().
241+ Str("provider", "ollama").
242+ Int("status", resp.StatusCode).
243+ Int("attempt", attempt+1).
244+ Str("body", string(payload)).
245+ Msg("Ollama retryable error")
246+ continue
247+ }
248+
249+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {
250+ payload, _ := io.ReadAll(resp.Body)
251+ resp.Body.Close()
252+ return nil, fmt.Errorf("chat completion status %d: %s", resp.StatusCode, strings.TrimSpace(string(payload)))
253+ }
254+
255+ bodyBytes, err := io.ReadAll(resp.Body)
256+ resp.Body.Close()
257+ if err != nil {
258+ return nil, fmt.Errorf("read response body: %w", err)
259+ }
260+
261+ var decoded chatCompletionResponse
262+ if err := json.Unmarshal(bodyBytes, &decoded); err != nil {
263+ return nil, fmt.Errorf("decode response: %w", err)
264+ }
265+
266+ return &decoded, nil
226267 }
227268
228- return &decoded, nil
269+ log.Error().
270+ Str("provider", "ollama").
271+ Int("max_retries", maxRetries).
272+ Err(lastErr).
273+ Msg("Ollama request failed after all retries")
274+ return nil, fmt.Errorf("request failed after %d retries: %w", maxRetries, lastErr)
229275 }
230276
231277 // Stream sends messages and returns a channel that streams response chunks.
internal/provider/opencode.go+46−17
@@ -37,6 +37,25 @@ type OpenCodeProvider struct {
3737 limiter *rate.Limiter
3838 }
3939
40+var opencodeRetryDelays = []time.Duration{5 * time.Second, 10 * time.Second, 15 * time.Second}
41+
42+const (
43+ opencodeChatCompletionsEndpoint = "/chat/completions"
44+ opencodeMessagesEndpoint = "/messages"
45+ opencodeResponsesEndpoint = "/responses"
46+)
47+
48+var opencodeModelEndpoints = map[string]string{
49+ "big-pickle": opencodeChatCompletionsEndpoint,
50+ "gemini-3-pro": "/models/gemini-3-pro",
51+ "gemini-3-flash": "/models/gemini-3-flash",
52+ "glm-4.7-free": opencodeChatCompletionsEndpoint,
53+ "gpt-5-nano": opencodeResponsesEndpoint,
54+ "kimi-k2.5-free": opencodeChatCompletionsEndpoint,
55+ "minimax-m2.1-free": opencodeMessagesEndpoint,
56+ "trinity-large-preview-free": opencodeChatCompletionsEndpoint,
57+}
58+
4059 // NewOpenCode creates a new OpenCode Zen provider.
4160 func NewOpenCode(endpoint, model, apiKey string) *OpenCodeProvider {
4261 return NewOpenCodeWithTemp(endpoint, model, apiKey, 0.7, nil)
@@ -65,12 +84,6 @@ func (p *OpenCodeProvider) Name() string {
6584
6685 // Chat sends messages and returns the complete response.
6786 func (p *OpenCodeProvider) Chat(ctx context.Context, messages []Message) (string, error) {
68- if p.limiter != nil {
69- if err := p.limiter.Wait(ctx); err != nil {
70- return "", err
71- }
72- }
73-
7487 resp, err := p.createChatCompletion(ctx, openai.ChatCompletionRequest{
7588 Model: p.model,
7689 Messages: mergeSystemMessagesOpenAI(toOpenAIMessages(messages)),
@@ -90,12 +103,6 @@ func (p *OpenCodeProvider) Chat(ctx context.Context, messages []Message) (string
90103
91104 // ChatWithTools sends messages with available tools and returns response with potential tool calls.
92105 func (p *OpenCodeProvider) ChatWithTools(ctx context.Context, messages []Message, tools []Tool) (*ChatResponse, error) {
93- if p.limiter != nil {
94- if err := p.limiter.Wait(ctx); err != nil {
95- return nil, err
96- }
97- }
98-
99106 // Convert tools to OpenAI format
100107 openaiTools, err := toOpenAITools(tools)
101108 if err != nil {
@@ -167,17 +174,15 @@ func (p *OpenCodeProvider) createChatCompletion(ctx context.Context, req openai.
167174 return nil, err
168175 }
169176
170- url := p.baseURL + "/chat/completions"
Cut short: this commit's diff is over 1000 lines.