@@ -0,0 +1,578 @@
1+package core
2+
3+import (
4+ "fmt"
5+ "path/filepath"
6+ "sync"
7+ "testing"
8+ "time"
9+
10+ "github.com/xonecas/zoea-nova/internal/config"
11+ "github.com/xonecas/zoea-nova/internal/provider"
12+ "github.com/xonecas/zoea-nova/internal/store"
13+)
14+
15+// TestLongRunningStopScenario reproduces the EXACT production failure:
16+// 1. App running for ~9 hours
17+// 2. Multiple myses active (long, sleepy runner)
18+// 3. User quits app (calls StopAll)
19+// 4. App should exit cleanly
20+// 5. All myses should show state=stopped (NOT errored)
21+//
22+// This test validates:
23+// - StopAll completes within 10 seconds
24+// - Both myses reach state=stopped (NOT errored)
25+// - No goroutine leaks (WaitGroup completes)
26+// - No deadlocks
27+// - lastError == nil for all myses
28+//
29+// Run with:
30+//
31+// go test ./internal/core -run TestLongRunningStopScenario -v -count=10
32+func TestLongRunningStopScenario(t *testing.T) {
33+ tmpDir := t.TempDir()
34+ dbPath := filepath.Join(tmpDir, "test.db")
35+ s, err := store.Open(dbPath)
36+ if err != nil {
37+ t.Fatalf("Open() error: %v", err)
38+ }
39+ defer s.Close()
40+
41+ bus := NewEventBus(100)
42+ defer bus.Close()
43+
44+ reg := provider.NewRegistry()
45+ cfg := &config.Config{
46+ Swarm: config.SwarmConfig{MaxMyses: 10},
47+ Providers: map[string]config.ProviderConfig{
48+ "mock": {Model: "test-model", Temperature: 0.7},
49+ },
50+ }
51+
52+ cmd := NewCommander(s, reg, bus, cfg)
53+
54+ // Create two myses: "long" and "sleepy"
55+ longMock := provider.NewMock("mock-long", "response from long")
56+ sleepyMock := provider.NewMock("mock-sleepy", "response from sleepy")
57+
58+ storedLong, _ := s.CreateMysis("long", "mock", "test-model", 0.7)
59+ storedSleepy, _ := s.CreateMysis("sleepy", "mock", "test-model", 0.7)
60+
61+ long := NewMysis(storedLong.ID, storedLong.Name, storedLong.CreatedAt, longMock, s, bus, cmd)
62+ sleepy := NewMysis(storedSleepy.ID, storedSleepy.Name, storedSleepy.CreatedAt, sleepyMock, s, bus, cmd)
63+
64+ // Add them to commander
65+ cmd.mu.Lock()
66+ cmd.myses[storedLong.ID] = long
67+ cmd.myses[storedSleepy.ID] = sleepy
68+ cmd.mu.Unlock()
69+
70+ // Start both myses
71+ if err := long.Start(); err != nil {
72+ t.Fatalf("Start(long) error: %v", err)
73+ }
74+ if err := sleepy.Start(); err != nil {
75+ t.Fatalf("Start(sleepy) error: %v", err)
76+ }
77+
78+ // Simulate long runtime with multiple turns
79+ // In production: 9 hours of runtime = hundreds of turns
80+ // In test: simulate with rapid-fire messages
81+ for i := 0; i < 5; i++ {
82+ long.SendMessage(fmt.Sprintf("message %d", i), store.MemorySourceDirect)
83+ sleepy.SendMessage(fmt.Sprintf("message %d", i), store.MemorySourceDirect)
84+ time.Sleep(50 * time.Millisecond) // Let messages process
85+ }
86+
87+ // Give myses time to settle into idle state
88+ time.Sleep(200 * time.Millisecond)
89+
90+ // Verify both are running before stop
91+ if long.State() != MysisStateRunning {
92+ t.Fatalf("expected long state=running before stop, got %s", long.State())
93+ }
94+ if sleepy.State() != MysisStateRunning {
95+ t.Fatalf("expected sleepy state=running before stop, got %s", sleepy.State())
96+ }
97+
98+ // CRITICAL: Call StopAll() - this is what TUI does on quit
99+ // This must complete within 10 seconds
100+ start := time.Now()
101+ cmd.StopAll()
102+ elapsed := time.Since(start)
103+
104+ // Verify StopAll completed within timeout
105+ if elapsed > 12*time.Second { // Allow 2s buffer
106+ t.Errorf("StopAll took %s (expected < 10s)", elapsed)
107+ }
108+
109+ // CRITICAL ASSERTION: Both myses should be in Stopped state (NOT Errored)
110+ if long.State() != MysisStateStopped {
111+ t.Errorf("PRODUCTION BUG REPRODUCED: long state=%s (expected stopped), lastError=%v",
112+ long.State(), long.LastError())
113+ }
114+ if sleepy.State() != MysisStateStopped {
115+ t.Errorf("PRODUCTION BUG REPRODUCED: sleepy state=%s (expected stopped), lastError=%v",
116+ sleepy.State(), sleepy.LastError())
117+ }
118+
119+ // Verify no errors were set
120+ if long.LastError() != nil {
121+ t.Errorf("long should have no error after clean stop, got: %v", long.LastError())
122+ }
123+ if sleepy.LastError() != nil {
124+ t.Errorf("sleepy should have no error after clean stop, got: %v", sleepy.LastError())
125+ }
126+
127+ // Verify WaitGroup completed (no goroutine leaks)
128+ // This is implicit in StopAll() completing, but verify explicitly
129+ done := make(chan struct{})
130+ go func() {
131+ cmd.wg.Wait()
132+ close(done)
133+ }()
134+
135+ select {
136+ case <-done:
137+ // WaitGroup completed - no goroutine leaks
138+ case <-time.After(1 * time.Second):
139+ t.Error("WaitGroup did not complete - goroutine leak detected")
140+ }
141+}
142+
143+// TestMultipleMysesStoppingSimultaneously tests the scenario where
144+// multiple myses are stopped at exactly the same time.
145+//
146+// This validates:
147+// - No race conditions in Stop()
148+// - No deadlocks when multiple Stop() calls happen simultaneously
149+// - All myses reach state=stopped
150+//
151+// Run with:
152+//
153+// go test ./internal/core -run TestMultipleMysesStoppingSimultaneously -v -count=10
154+func TestMultipleMysesStoppingSimultaneously(t *testing.T) {
155+ s, bus, cleanup := setupMysisTest(t)
156+ defer cleanup()
157+
158+ reg := provider.NewRegistry()
159+ cfg := &config.Config{
160+ Swarm: config.SwarmConfig{MaxMyses: 10},
161+ Providers: map[string]config.ProviderConfig{
162+ "mock": {Model: "test-model", Temperature: 0.7},
163+ },
164+ }
165+
166+ cmd := NewCommander(s, reg, bus, cfg)
167+
168+ // Create 5 myses
169+ myses := make([]*Mysis, 5)
170+ for i := 0; i < 5; i++ {
171+ mock := provider.NewMock(fmt.Sprintf("mock-%d", i), "response")
172+ stored, _ := s.CreateMysis(fmt.Sprintf("mysis-%d", i), "mock", "test-model", 0.7)
173+ mysis := NewMysis(stored.ID, stored.Name, stored.CreatedAt, mock, s, bus, cmd)
174+ myses[i] = mysis
175+
176+ cmd.mu.Lock()
177+ cmd.myses[stored.ID] = mysis
178+ cmd.mu.Unlock()
179+
180+ if err := mysis.Start(); err != nil {
181+ t.Fatalf("Start(mysis-%d) error: %v", i, err)
182+ }
183+ }
184+
185+ // Wait for myses to stabilize
186+ time.Sleep(100 * time.Millisecond)
187+
188+ // Stop all myses simultaneously
189+ var wg sync.WaitGroup
190+ errors := make([]error, 5)
191+
192+ for i := 0; i < 5; i++ {
193+ wg.Add(1)
194+ go func(idx int) {
195+ defer wg.Done()
196+ errors[idx] = myses[idx].Stop()
197+ }(i)
198+ }
199+
200+ // Wait for all stops to complete
201+ done := make(chan struct{})
202+ go func() {
203+ wg.Wait()
204+ close(done)
205+ }()
206+
207+ select {
208+ case <-done:
209+ // All stops completed
210+ case <-time.After(5 * time.Second):
211+ t.Fatal("timeout waiting for simultaneous stops - possible deadlock")
212+ }
213+
214+ // Verify all stopped successfully
215+ for i := 0; i < 5; i++ {
216+ if errors[i] != nil {
217+ t.Errorf("mysis-%d Stop() error: %v", i, errors[i])
218+ }
219+ if myses[i].State() != MysisStateStopped {
220+ t.Errorf("mysis-%d state=%s (expected stopped), lastError=%v",
221+ i, myses[i].State(), myses[i].LastError())
222+ }
223+ if myses[i].LastError() != nil {
224+ t.Errorf("mysis-%d should have no error, got: %v", i, myses[i].LastError())
225+ }
226+ }
227+}
228+
229+// TestStopDuringActiveLLMCalls tests stopping myses while they are
230+// actively processing LLM calls.
231+//
232+// This validates:
233+// - Stop() waits for active LLM calls to complete
234+// - Context cancellation is handled gracefully
235+// - Final state is stopped (NOT errored)
236+//
237+// Run with:
238+//
239+// go test ./internal/core -run TestStopDuringActiveLLMCalls -v -count=10
240+func TestStopDuringActiveLLMCalls(t *testing.T) {
241+ s, bus, cleanup := setupMysisTest(t)
242+ defer cleanup()
243+
244+ stored, _ := s.CreateMysis("active-call", "mock", "test-model", 0.7)
245+
246+ // Mock with delay to simulate active LLM call
247+ mock := provider.NewMock("mock", "response").SetDelay(200 * time.Millisecond)
248+ mysis := NewMysis(stored.ID, stored.Name, stored.CreatedAt, mock, s, bus)
249+
250+ if err := mysis.Start(); err != nil {
251+ t.Fatalf("Start() error: %v", err)
252+ }
253+
254+ // Wait for initial autonomous turn to complete
255+ time.Sleep(100 * time.Millisecond)
256+
257+ // Send a message that will take 200ms to process
258+ messageDone := make(chan struct{})
259+ go func() {
260+ _ = mysis.SendMessage("slow message", store.MemorySourceDirect)
261+ close(messageDone)
262+ }()
263+
264+ // Wait to ensure message processing started
265+ time.Sleep(100 * time.Millisecond)
266+
267+ // Stop while LLM is processing
268+ start := time.Now()
269+ if err := mysis.Stop(); err != nil {
270+ t.Fatalf("Stop() error: %v", err)
271+ }
272+ elapsed := time.Since(start)
273+
274+ // Stop should complete within reasonable time
275+ // It may wait for the current turn or cancel immediately - both are acceptable
276+ if elapsed > 1*time.Second {
277+ t.Errorf("Stop took too long (%s) - possible deadlock", elapsed)
278+ }
279+
280+ // Verify final state
281+ if mysis.State() != MysisStateStopped {
282+ t.Errorf("expected state=stopped, got %s (lastError=%v)", mysis.State(), mysis.LastError())
283+ }
284+ if mysis.LastError() != nil {
285+ t.Errorf("expected no error after Stop during active call, got: %v", mysis.LastError())
286+ }
287+}
288+
289+// TestStopWithQueuedMessages tests stopping a mysis that has queued messages
290+// waiting to be processed.
291+//
292+// This validates:
293+// - Queued messages are not processed after Stop()
294+// - No deadlock when stopping with queued messages
295+// - Final state is stopped
296+//
297+// Run with:
298+//
299+// go test ./internal/core -run TestStopWithQueuedMessages -v -count=10
300+func TestStopWithQueuedMessages(t *testing.T) {
301+ s, bus, cleanup := setupMysisTest(t)
302+ defer cleanup()
303+
304+ stored, _ := s.CreateMysis("queued", "mock", "test-model", 0.7)
305+
306+ // Mock with delay to create queue pressure
307+ mock := provider.NewMock("mock", "response").SetDelay(100 * time.Millisecond)
308+ mysis := NewMysis(stored.ID, stored.Name, stored.CreatedAt, mock, s, bus)
309+
310+ if err := mysis.Start(); err != nil {
311+ t.Fatalf("Start() error: %v", err)
312+ }
313+
Cut short: this commit's diff is over 1000 lines.