@@ -1,583 +0,0 @@
1-package core
2-
3-import (
4- "fmt"
5- "path/filepath"
6- "strings"
7- "sync"
8- "testing"
9- "time"
10-
11- "github.com/xonecas/zoea-nova/internal/config"
12- "github.com/xonecas/zoea-nova/internal/provider"
13- "github.com/xonecas/zoea-nova/internal/store"
14-)
15-
16-// TestLongRunningStopScenario reproduces the EXACT production failure:
17-// 1. App running for ~9 hours
18-// 2. Multiple myses active (long, sleepy runner)
19-// 3. User quits app (calls StopAll)
20-// 4. App should exit cleanly
21-// 5. All myses should show state=stopped (NOT errored)
22-//
23-// This test validates:
24-// - StopAll completes within 10 seconds
25-// - Both myses reach state=stopped (NOT errored)
26-// - No goroutine leaks (WaitGroup completes)
27-// - No deadlocks
28-// - lastError == nil for all myses
29-//
30-// Run with:
31-//
32-// go test ./internal/core -run TestLongRunningStopScenario -v -count=10
33-func TestLongRunningStopScenario(t *testing.T) {
34- tmpDir := t.TempDir()
35- dbPath := filepath.Join(tmpDir, "test.db")
36- s, err := store.Open(dbPath)
37- if err != nil {
38- t.Fatalf("Open() error: %v", err)
39- }
40- defer s.Close()
41-
42- bus := NewEventBus(100)
43- defer bus.Close()
44-
45- reg := provider.NewRegistry()
46- cfg := &config.Config{
47- Swarm: config.SwarmConfig{MaxMyses: 10},
48- Providers: map[string]config.ProviderConfig{
49- "mock": {Model: "test-model", Temperature: 0.7},
50- },
51- }
52-
53- cmd := NewCommander(s, reg, bus, cfg, "")
54-
55- // Create two myses: "long" and "sleepy"
56- longMock := provider.NewMock("mock-long", "response from long")
57- sleepyMock := provider.NewMock("mock-sleepy", "response from sleepy")
58-
59- storedLong, _ := s.CreateMysis("long", "mock", "test-model", 0.7)
60- storedSleepy, _ := s.CreateMysis("sleepy", "mock", "test-model", 0.7)
61-
62- long := NewMysis(storedLong.ID, storedLong.Name, storedLong.CreatedAt, longMock, s, bus, "", cmd)
63- sleepy := NewMysis(storedSleepy.ID, storedSleepy.Name, storedSleepy.CreatedAt, sleepyMock, s, bus, "", cmd)
64-
65- // Add them to commander
66- cmd.mu.Lock()
67- cmd.myses[storedLong.ID] = long
68- cmd.myses[storedSleepy.ID] = sleepy
69- cmd.mu.Unlock()
70-
71- // Start both myses
72- if err := long.Start(); err != nil {
73- t.Fatalf("Start(long) error: %v", err)
74- }
75- if err := sleepy.Start(); err != nil {
76- t.Fatalf("Start(sleepy) error: %v", err)
77- }
78-
79- // Simulate long runtime with multiple turns
80- // In production: 9 hours of runtime = hundreds of turns
81- // In test: simulate with rapid-fire messages
82- for i := 0; i < 5; i++ {
83- long.SendMessage(fmt.Sprintf("message %d", i), store.MemorySourceDirect)
84- sleepy.SendMessage(fmt.Sprintf("message %d", i), store.MemorySourceDirect)
85- time.Sleep(50 * time.Millisecond) // Let messages process
86- }
87-
88- // Give myses time to settle into idle state
89- time.Sleep(200 * time.Millisecond)
90-
91- // Verify both are running before stop
92- if long.State() != MysisStateRunning {
93- t.Fatalf("expected long state=running before stop, got %s", long.State())
94- }
95- if sleepy.State() != MysisStateRunning {
96- t.Fatalf("expected sleepy state=running before stop, got %s", sleepy.State())
97- }
98-
99- // CRITICAL: Call StopAll() - this is what TUI does on quit
100- // This must complete within 10 seconds
101- start := time.Now()
102- cmd.StopAll()
103- elapsed := time.Since(start)
104-
105- // Verify StopAll completed within timeout
106- if elapsed > 12*time.Second { // Allow 2s buffer
107- t.Errorf("StopAll took %s (expected < 10s)", elapsed)
108- }
109-
110- // CRITICAL ASSERTION: Both myses should be in Stopped state (NOT Errored)
111- if long.State() != MysisStateStopped {
112- t.Errorf("PRODUCTION BUG REPRODUCED: long state=%s (expected stopped), lastError=%v",
113- long.State(), long.LastError())
114- }
115- if sleepy.State() != MysisStateStopped {
116- t.Errorf("PRODUCTION BUG REPRODUCED: sleepy state=%s (expected stopped), lastError=%v",
117- sleepy.State(), sleepy.LastError())
118- }
119-
120- // Verify no errors were set
121- if long.LastError() != nil {
122- t.Errorf("long should have no error after clean stop, got: %v", long.LastError())
123- }
124- if sleepy.LastError() != nil {
125- t.Errorf("sleepy should have no error after clean stop, got: %v", sleepy.LastError())
126- }
127-
128- // Verify WaitGroup completed (no goroutine leaks)
129- // This is implicit in StopAll() completing, but verify explicitly
130- done := make(chan struct{})
131- go func() {
132- cmd.wg.Wait()
133- close(done)
134- }()
135-
136- select {
137- case <-done:
138- // WaitGroup completed - no goroutine leaks
139- case <-time.After(1 * time.Second):
140- t.Error("WaitGroup did not complete - goroutine leak detected")
141- }
142-}
143-
144-// TestMultipleMysesStoppingSimultaneously tests the scenario where
145-// multiple myses are stopped at exactly the same time.
146-//
147-// This validates:
148-// - No race conditions in Stop()
149-// - No deadlocks when multiple Stop() calls happen simultaneously
150-// - All myses reach state=stopped
151-//
152-// Run with:
153-//
154-// go test ./internal/core -run TestMultipleMysesStoppingSimultaneously -v -count=10
155-func TestMultipleMysesStoppingSimultaneously(t *testing.T) {
156- s, bus, cleanup := setupMysisTest(t)
157- defer cleanup()
158-
159- reg := provider.NewRegistry()
160- cfg := &config.Config{
161- Swarm: config.SwarmConfig{MaxMyses: 10},
162- Providers: map[string]config.ProviderConfig{
163- "mock": {Model: "test-model", Temperature: 0.7},
164- },
165- }
166-
167- cmd := NewCommander(s, reg, bus, cfg, "")
168-
169- // Create 5 myses
170- myses := make([]*Mysis, 5)
171- for i := 0; i < 5; i++ {
172- mock := provider.NewMock(fmt.Sprintf("mock-%d", i), "response")
173- stored, _ := s.CreateMysis(fmt.Sprintf("mysis-%d", i), "mock", "test-model", 0.7)
174- mysis := NewMysis(stored.ID, stored.Name, stored.CreatedAt, mock, s, bus, "", cmd)
175- myses[i] = mysis
176-
177- cmd.mu.Lock()
178- cmd.myses[stored.ID] = mysis
179- cmd.mu.Unlock()
180-
181- if err := mysis.Start(); err != nil {
182- t.Fatalf("Start(mysis-%d) error: %v", i, err)
183- }
184- }
185-
186- // Wait for myses to stabilize
187- time.Sleep(100 * time.Millisecond)
188-
189- // Stop all myses simultaneously
190- var wg sync.WaitGroup
191- errors := make([]error, 5)
192-
193- for i := 0; i < 5; i++ {
194- wg.Add(1)
195- go func(idx int) {
196- defer wg.Done()
197- errors[idx] = myses[idx].Stop()
198- }(i)
199- }
200-
201- // Wait for all stops to complete
202- done := make(chan struct{})
203- go func() {
204- wg.Wait()
205- close(done)
206- }()
207-
208- select {
209- case <-done:
210- // All stops completed
211- case <-time.After(5 * time.Second):
212- t.Fatal("timeout waiting for simultaneous stops - possible deadlock")
213- }
214-
215- // Verify all stopped successfully
216- for i := 0; i < 5; i++ {
217- if errors[i] != nil {
218- t.Errorf("mysis-%d Stop() error: %v", i, errors[i])
219- }
220- if myses[i].State() != MysisStateStopped {
221- t.Errorf("mysis-%d state=%s (expected stopped), lastError=%v",
222- i, myses[i].State(), myses[i].LastError())
223- }
224- if myses[i].LastError() != nil {
225- t.Errorf("mysis-%d should have no error, got: %v", i, myses[i].LastError())
226- }
227- }
228-}
229-
230-// TestStopDuringActiveLLMCalls tests stopping myses while they are
231-// actively processing LLM calls.
232-//
233-// This validates:
234-// - Stop() waits for active LLM calls to complete
235-// - Context cancellation is handled gracefully
236-// - Final state is stopped (NOT errored)
237-//
238-// Run with:
239-//
240-// go test ./internal/core -run TestStopDuringActiveLLMCalls -v -count=10
241-func TestStopDuringActiveLLMCalls(t *testing.T) {
242- s, bus, cleanup := setupMysisTest(t)
243- defer cleanup()
244-
245- stored, _ := s.CreateMysis("active-call", "mock", "test-model", 0.7)
246-
247- // Mock with delay to simulate active LLM call
248- mock := provider.NewMock("mock", "response").SetDelay(200 * time.Millisecond)
249- mysis := NewMysis(stored.ID, stored.Name, stored.CreatedAt, mock, s, bus, "")
250-
251- if err := mysis.Start(); err != nil {
252- t.Fatalf("Start() error: %v", err)
253- }
254-
255- // Wait for initial turn (triggered by encouragement message) to complete
256- time.Sleep(100 * time.Millisecond)
257-
258- // Send a message that will take 200ms to process
259- messageDone := make(chan struct{})
260- go func() {
261- _ = mysis.SendMessage("slow message", store.MemorySourceDirect)
262- close(messageDone)
263- }()
264-
265- // Wait to ensure message processing started
266- time.Sleep(100 * time.Millisecond)
267-
268- // Stop while LLM is processing
269- start := time.Now()
270- if err := mysis.Stop(); err != nil {
271- t.Fatalf("Stop() error: %v", err)
272- }
273- elapsed := time.Since(start)
274-
275- // Stop should complete within reasonable time
276- // It may wait for the current turn or cancel immediately - both are acceptable
277- if elapsed > 1*time.Second {
278- t.Errorf("Stop took too long (%s) - possible deadlock", elapsed)
279- }
280-
281- // Verify final state
282- if mysis.State() != MysisStateStopped {
283- t.Errorf("expected state=stopped, got %s (lastError=%v)", mysis.State(), mysis.LastError())
284- }
285- if mysis.LastError() != nil {
286- t.Errorf("expected no error after Stop during active call, got: %v", mysis.LastError())
287- }
288-}
289-
290-// TestStopWithQueuedMessages tests stopping a mysis that has queued messages
291-// waiting to be processed.
292-//
293-// This validates:
294-// - Queued messages are not processed after Stop()
295-// - No deadlock when stopping with queued messages
296-// - Final state is stopped
297-//
298-// Run with:
299-//
300-// go test ./internal/core -run TestStopWithQueuedMessages -v -count=10
301-func TestStopWithQueuedMessages(t *testing.T) {
302- s, bus, cleanup := setupMysisTest(t)
303- defer cleanup()
304-
305- stored, _ := s.CreateMysis("queued", "mock", "test-model", 0.7)
306-
307- // Mock with delay to create queue pressure
308- mock := provider.NewMock("mock", "response").SetDelay(100 * time.Millisecond)
309- mysis := NewMysis(stored.ID, stored.Name, stored.CreatedAt, mock, s, bus, "")
310-
311- if err := mysis.Start(); err != nil {
312- t.Fatalf("Start() error: %v", err)
313- }
314-
315- // Queue multiple messages rapidly
316- // Only one will process at a time due to turnMu
317- for i := 0; i < 5; i++ {
318- go func(idx int) {
319- _ = mysis.SendMessage(fmt.Sprintf("queued-%d", idx), store.MemorySourceDirect)
320- }(i)
321- }
322-
323- // Wait a bit to let some messages queue
324- time.Sleep(50 * time.Millisecond)
325-
326- // Stop while messages are queued
327- if err := mysis.Stop(); err != nil {
328- t.Fatalf("Stop() error: %v", err)
329- }
330-
331- // Verify final state
332- if mysis.State() != MysisStateStopped {
333- t.Errorf("expected state=stopped, got %s (lastError=%v)", mysis.State(), mysis.LastError())
334- }
335- if mysis.LastError() != nil {
336- t.Errorf("expected no error, got: %v", mysis.LastError())
337- }
338-
339- // Verify no further messages can be sent
340- err := mysis.SendMessage("after-stop", store.MemorySourceDirect)
341- if err == nil {
342- t.Error("expected error sending message to stopped mysis")
343- } else {
344- if !strings.Contains(err.Error(), "stopped") || !strings.Contains(err.Error(), "relaunch") {
345- t.Errorf("expected error about stopped state requiring relaunch, got: %v", err)
346- }
347- }
348-}
349-
350-// TestCleanExitAfterStop tests that the app can exit cleanly after StopAll.
351-//
352-// This validates:
353-// - EventBus can be closed after StopAll
354-// - Store can be closed after StopAll
355-// - No goroutines block cleanup
356-// - No panic during cleanup
357-//
358-// Run with:
359-//
360-// go test ./internal/core -run TestCleanExitAfterStop -v -count=10
361-func TestCleanExitAfterStop(t *testing.T) {
362- tmpDir := t.TempDir()
363- dbPath := filepath.Join(tmpDir, "test.db")
364- s, err := store.Open(dbPath)
365- if err != nil {
366- t.Fatalf("Open() error: %v", err)
367- }
368-
369- bus := NewEventBus(100)
370-
371- reg := provider.NewRegistry()
372- cfg := &config.Config{
373- Swarm: config.SwarmConfig{MaxMyses: 10},
374- Providers: map[string]config.ProviderConfig{
375- "mock": {Model: "test-model", Temperature: 0.7},
376- },
377- }
378-
379- cmd := NewCommander(s, reg, bus, cfg, "")
380-
381- // Create and start multiple myses
382- for i := 0; i < 3; i++ {
383- mock := provider.NewMock(fmt.Sprintf("mock-%d", i), "response")
384- stored, _ := s.CreateMysis(fmt.Sprintf("exit-test-%d", i), "mock", "test-model", 0.7)
385- mysis := NewMysis(stored.ID, stored.Name, stored.CreatedAt, mock, s, bus, "", cmd)
386-
387- cmd.mu.Lock()
388- cmd.myses[stored.ID] = mysis
389- cmd.mu.Unlock()
390-
391- if err := mysis.Start(); err != nil {
392- t.Fatalf("Start() error: %v", err)
393- }
394- }
395-
396- // Wait for myses to stabilize
397- time.Sleep(100 * time.Millisecond)
398-
399- // Stop all myses
400- cmd.StopAll()
401-
402- // Now perform cleanup (this is what main.go does)
403- // If this hangs or panics, we have a problem
404- done := make(chan struct{})
405- go func() {
406- bus.Close()
407- s.Close()
408- close(done)
409- }()
410-
411- select {
412- case <-done:
413- // Clean exit successful
414- case <-time.After(2 * time.Second):
415- t.Fatal("cleanup hung - possible goroutine leak or deadlock")
416- }
417-}
418-
419-// TestStopAllWithMixedStates tests StopAll when myses are in different states.
420-//
421-// This validates:
422-// - StopAll handles idle myses correctly
423-// - StopAll skips already-stopped myses
424-// - StopAll doesn't affect errored myses
425-// - No panic when calling Stop on non-running myses
426-//
427-// Run with:
428-//
429-// go test ./internal/core -run TestStopAllWithMixedStates -v -count=10
430-func TestStopAllWithMixedStates(t *testing.T) {
431- s, bus, cleanup := setupMysisTest(t)
432- defer cleanup()
433-
434- reg := provider.NewRegistry()
435- cfg := &config.Config{
436- Swarm: config.SwarmConfig{MaxMyses: 10},
437- Providers: map[string]config.ProviderConfig{
438- "mock": {Model: "test-model", Temperature: 0.7},
439- },
440- }
441-
442- cmd := NewCommander(s, reg, bus, cfg, "")
443-
444- // Create myses in different states
445- // 1. Running
446- runningStored, _ := s.CreateMysis("running", "mock", "test-model", 0.7)
447- runningMock := provider.NewMock("mock-running", "response")
448- running := NewMysis(runningStored.ID, runningStored.Name, runningStored.CreatedAt, runningMock, s, bus, "", cmd)
449- cmd.mu.Lock()
450- cmd.myses[runningStored.ID] = running
451- cmd.mu.Unlock()
452- running.Start()
453-
454- // 2. Idle (never started)
455- idleStored, _ := s.CreateMysis("idle", "mock", "test-model", 0.7)
456- idleMock := provider.NewMock("mock-idle", "response")
457- idle := NewMysis(idleStored.ID, idleStored.Name, idleStored.CreatedAt, idleMock, s, bus, "", cmd)
458- cmd.mu.Lock()
459- cmd.myses[idleStored.ID] = idle
460- cmd.mu.Unlock()
461-
462- // 3. Already stopped
463- stoppedStored, _ := s.CreateMysis("stopped", "mock", "test-model", 0.7)
464- stoppedMock := provider.NewMock("mock-stopped", "response")
465- stopped := NewMysis(stoppedStored.ID, stoppedStored.Name, stoppedStored.CreatedAt, stoppedMock, s, bus, "", cmd)
466- cmd.mu.Lock()
467- cmd.myses[stoppedStored.ID] = stopped
468- cmd.mu.Unlock()
469- stopped.Start()
470- stopped.Stop()
471-
472- // Wait for states to stabilize
473- time.Sleep(100 * time.Millisecond)
474-
475- // Call StopAll - should handle mixed states gracefully
476- start := time.Now()
477- cmd.StopAll()
478- elapsed := time.Since(start)
479-
480- if elapsed > 5*time.Second {
481- t.Errorf("StopAll took %s (expected < 5s)", elapsed)
482- }
483-
484- // Verify states
485- if running.State() != MysisStateStopped {
486- t.Errorf("running mysis state=%s (expected stopped)", running.State())
487- }
488- if idle.State() != MysisStateIdle {
489- t.Errorf("idle mysis state=%s (expected idle)", idle.State())
490- }
491- if stopped.State() != MysisStateStopped {
492- t.Errorf("stopped mysis state=%s (expected stopped)", stopped.State())
493- }
494-
495- // Verify no errors
496- if running.LastError() != nil {
497- t.Errorf("running mysis error: %v", running.LastError())
498- }
499- if idle.LastError() != nil {
500- t.Errorf("idle mysis error: %v", idle.LastError())
501- }
502- if stopped.LastError() != nil {
503- t.Errorf("stopped mysis error: %v", stopped.LastError())
504- }
505-}
506-
507-// TestStopAllStressTest performs repeated StopAll operations to detect
508-// rare race conditions and timing issues.
509-//
510-// Run with:
511-//
512-// go test ./internal/core -run TestStopAllStressTest -v -count=20
513-func TestStopAllStressTest(t *testing.T) {
514- for iteration := 0; iteration < 5; iteration++ {
515- t.Run(fmt.Sprintf("iteration_%d", iteration), func(t *testing.T) {
516- tmpDir := t.TempDir()
517- dbPath := filepath.Join(tmpDir, "test.db")
518- s, err := store.Open(dbPath)
519- if err != nil {
520- t.Fatalf("Open() error: %v", err)
521- }
522- defer s.Close()
523-
524- bus := NewEventBus(100)
525- defer bus.Close()
526-
527- reg := provider.NewRegistry()
528- cfg := &config.Config{
529- Swarm: config.SwarmConfig{MaxMyses: 10},
530- Providers: map[string]config.ProviderConfig{
531- "mock": {Model: "test-model", Temperature: 0.7},
532- },
533- }
534-
535- cmd := NewCommander(s, reg, bus, cfg, "")
536-
537- // Create multiple myses
538- for i := 0; i < 3; i++ {
539- mock := provider.NewMock(fmt.Sprintf("mock-%d", i), "response")
540- stored, _ := s.CreateMysis(fmt.Sprintf("stress-%d", i), "mock", "test-model", 0.7)
541- mysis := NewMysis(stored.ID, stored.Name, stored.CreatedAt, mock, s, bus, "", cmd)
542-
543- cmd.mu.Lock()
544- cmd.myses[stored.ID] = mysis
545- cmd.mu.Unlock()
546-
547- if err := mysis.Start(); err != nil {
548- t.Fatalf("Start() error: %v", err)
549- }
550-
551- // Send some messages to simulate activity
552- go func() {
553- _ = mysis.SendMessage("stress test", store.MemorySourceDirect)
554- }()
555- }
556-
557- // Random sleep to vary timing
558- time.Sleep(time.Duration(50+iteration*20) * time.Millisecond)
559-
560- // StopAll
561- start := time.Now()
562- cmd.StopAll()
563- elapsed := time.Since(start)
564-
565- if elapsed > 10*time.Second {
566- t.Errorf("iteration %d: StopAll timeout (%s)", iteration, elapsed)
567- }
568-
569- // Verify all myses are stopped
570- cmd.mu.RLock()
571- for id, mysis := range cmd.myses {
572- state := mysis.State()
573- // Running myses should now be stopped
574- // Idle myses remain idle
575- if state != MysisStateStopped && state != MysisStateIdle {
576- t.Errorf("iteration %d: mysis %s state=%s (expected stopped or idle), lastError=%v",
577- iteration, id, state, mysis.LastError())
578- }
579- }
580- cmd.mu.RUnlock()
581- })
582- }
583-}