xonecas.io
60255
Zn

Zoea-Nova / commit ca24980

fix(core): harden event bus and ticks

Use blocking publish for critical events, keep nudges async, and make tick detection more robust.

xonecas2026-02-05 ca249802ae61c1e22e56a7b60456f994aec3d739 parent 6e678ce

5 files changed

internal/constants/constants.go+3−0
@@ -145,6 +145,9 @@ const FallbackLLMResponse = "(no response)"
145145 // MinEventBusBufferSize is the minimum buffer per subscriber channel.
146146 const MinEventBusBufferSize = 1000
147147
148+// EventBusPublishTimeout is the per-subscriber timeout for critical events.
149+const EventBusPublishTimeout = 200 * time.Millisecond
150+
148151 // SnapshotTools defines tools that return state snapshots for context compaction.
149152 var SnapshotTools = map[string]bool{
150153 "get_ship": true,
internal/core/bus.go+125−13
@@ -2,14 +2,66 @@ package core
22
33 import (
44 "sync"
5+ "sync/atomic"
6+ "time"
57
8+ "github.com/rs/zerolog/log"
69 "github.com/xonecas/zoea-nova/internal/constants"
710 )
811
12+const dropLogEvery = 100
13+
14+type subscriber struct {
15+ mu sync.RWMutex
16+ ch chan Event
17+ dropped atomic.Uint64
18+ closed bool
19+}
20+
21+func (s *subscriber) send(event Event, timeout time.Duration) (bool, bool) {
22+ s.mu.RLock()
23+ defer s.mu.RUnlock()
24+
25+ if s.closed {
26+ return false, false
27+ }
28+
29+ if timeout <= 0 {
30+ select {
31+ case s.ch <- event:
32+ return true, false
33+ default:
34+ return false, false
35+ }
36+ }
37+
38+ timer := time.NewTimer(timeout)
39+ select {
40+ case s.ch <- event:
41+ if !timer.Stop() {
42+ <-timer.C
43+ }
44+ return true, false
45+ case <-timer.C:
46+ return false, true
47+ }
48+}
49+
50+func (s *subscriber) close() {
51+ s.mu.Lock()
52+ if s.closed {
53+ s.mu.Unlock()
54+ return
55+ }
56+ s.closed = true
57+ close(s.ch)
58+ s.mu.Unlock()
59+}
60+
961 // EventBus distributes events to subscribers.
1062 type EventBus struct {
1163 mu sync.RWMutex
12- subscribers []chan Event
64+ subscribers []*subscriber
1365 bufferSize int
1466 }
1567
@@ -29,9 +81,9 @@ func (b *EventBus) Subscribe() <-chan Event {
2981 b.mu.Lock()
3082 defer b.mu.Unlock()
3183
32- ch := make(chan Event, b.bufferSize)
33- b.subscribers = append(b.subscribers, ch)
34- return ch
84+ sub := &subscriber{ch: make(chan Event, b.bufferSize)}
85+ b.subscribers = append(b.subscribers, sub)
86+ return sub.ch
3587 }
3688
3789 // Unsubscribe removes a subscriber channel.
@@ -40,9 +92,9 @@ func (b *EventBus) Unsubscribe(ch <-chan Event) {
4092 defer b.mu.Unlock()
4193
4294 for i, sub := range b.subscribers {
43- if sub == ch {
44- close(sub)
95+ if sub.ch == ch {
4596 b.subscribers = append(b.subscribers[:i], b.subscribers[i+1:]...)
97+ sub.close()
4698 return
4799 }
48100 }
@@ -52,24 +104,84 @@ func (b *EventBus) Unsubscribe(ch <-chan Event) {
52104 // Non-blocking: drops events if a subscriber's buffer is full.
53105 func (b *EventBus) Publish(event Event) {
54106 b.mu.RLock()
55- defer b.mu.RUnlock()
107+ subscribers := append([]*subscriber(nil), b.subscribers...)
108+ b.mu.RUnlock()
56109
57- for _, ch := range b.subscribers {
58- select {
59- case ch <- event:
60- default:
61- // Drop event if buffer is full (non-blocking)
110+ for i, sub := range subscribers {
111+ if delivered, _ := sub.send(event, 0); !delivered {
112+ dropped := sub.dropped.Add(1)
113+ if dropped%dropLogEvery == 0 {
114+ log.Warn().
115+ Str("event_type", string(event.Type)).
116+ Uint64("dropped", dropped).
117+ Int("subscriber_index", i).
118+ Msg("event bus subscriber dropped events")
119+ }
62120 }
63121 }
64122 }
65123
124+// PublishBlocking sends an event to all subscribers, waiting up to timeout per subscriber.
125+// Returns true if all subscribers received the event.
126+func (b *EventBus) PublishBlocking(event Event, timeout time.Duration) bool {
127+ if timeout <= 0 {
128+ b.mu.RLock()
129+ subscribers := append([]*subscriber(nil), b.subscribers...)
130+ b.mu.RUnlock()
131+
132+ allDelivered := true
133+ for i, sub := range subscribers {
134+ if delivered, _ := sub.send(event, 0); !delivered {
135+ allDelivered = false
136+ dropped := sub.dropped.Add(1)
137+ if dropped%dropLogEvery == 0 {
138+ log.Warn().
139+ Str("event_type", string(event.Type)).
140+ Uint64("dropped", dropped).
141+ Int("subscriber_index", i).
142+ Msg("event bus subscriber dropped events")
143+ }
144+ }
145+ }
146+
147+ return allDelivered
148+ }
149+
150+ b.mu.RLock()
151+ subscribers := append([]*subscriber(nil), b.subscribers...)
152+ b.mu.RUnlock()
153+
154+ allDelivered := true
155+ for i, sub := range subscribers {
156+ delivered, timedOut := sub.send(event, timeout)
157+ if delivered {
158+ continue
159+ }
160+ allDelivered = false
161+ dropped := sub.dropped.Add(1)
162+ if dropped%dropLogEvery == 0 {
163+ msg := "event bus subscriber dropped events"
164+ if timedOut {
165+ msg = "event bus subscriber timed out"
166+ }
167+ log.Warn().
168+ Str("event_type", string(event.Type)).
169+ Uint64("dropped", dropped).
170+ Int("subscriber_index", i).
171+ Msg(msg)
172+ }
173+ }
174+
175+ return allDelivered
176+}
177+
66178 // Close closes all subscriber channels.
67179 func (b *EventBus) Close() {
68180 b.mu.Lock()
69181 defer b.mu.Unlock()
70182
71183 for _, ch := range b.subscribers {
72- close(ch)
184+ ch.close()
73185 }
74186 b.subscribers = nil
75187 }
internal/core/bus_test.go+68−2
@@ -77,8 +77,9 @@ func TestEventBusNonBlocking(t *testing.T) {
7777 bus := NewEventBus(1)
7878 ch := bus.Subscribe()
7979
80- // Fill buffer
81- bus.Publish(Event{Type: EventMysisCreated})
80+ for len(ch) < cap(ch) {
81+ bus.Publish(Event{Type: EventMysisCreated})
82+ }
8283
8384 // This should not block (event dropped)
8485 done := make(chan bool)
@@ -99,6 +100,71 @@ func TestEventBusNonBlocking(t *testing.T) {
99100 bus.Close()
100101 }
101102
103+func TestEventBusPublishBlockingDelivers(t *testing.T) {
104+ bus := NewEventBus(1)
105+ ch := bus.Subscribe()
106+
107+ for len(ch) < cap(ch) {
108+ bus.Publish(Event{Type: EventMysisCreated})
109+ }
110+
111+ resultCh := make(chan bool, 1)
112+ go func() {
113+ resultCh <- bus.PublishBlocking(Event{Type: EventMysisDeleted}, 100*time.Millisecond)
114+ }()
115+
116+ select {
117+ case <-resultCh:
118+ t.Fatal("expected PublishBlocking to wait for buffer space")
119+ case <-time.After(20 * time.Millisecond):
120+ }
121+
122+ <-ch
123+
124+ select {
125+ case delivered := <-resultCh:
126+ if !delivered {
127+ t.Fatal("expected PublishBlocking to deliver")
128+ }
129+ case <-time.After(200 * time.Millisecond):
130+ t.Fatal("timeout waiting for PublishBlocking")
131+ }
132+
133+ deletedFound := false
134+ for i := 0; i < cap(ch); i++ {
135+ received := <-ch
136+ if received.Type == EventMysisDeleted {
137+ deletedFound = true
138+ }
139+ }
140+ if !deletedFound {
141+ t.Fatal("expected to find blocking event")
142+ }
143+
144+ bus.Close()
145+}
146+
147+func TestEventBusPublishBlockingTimeout(t *testing.T) {
148+ bus := NewEventBus(1)
149+ ch := bus.Subscribe()
150+
151+ for len(ch) < cap(ch) {
152+ bus.Publish(Event{Type: EventMysisCreated})
153+ }
154+
155+ start := time.Now()
156+ delivered := bus.PublishBlocking(Event{Type: EventMysisDeleted}, 20*time.Millisecond)
157+ if delivered {
158+ t.Fatal("expected PublishBlocking to timeout")
159+ }
160+ if time.Since(start) < 15*time.Millisecond {
161+ t.Fatal("expected PublishBlocking to wait for timeout")
162+ }
163+
164+ <-ch
165+ bus.Close()
166+}
167+
102168 func TestEventBusClose(t *testing.T) {
103169 bus := NewEventBus(10)
104170
internal/core/mysis.go+121−25
@@ -4,6 +4,7 @@ import (
44 "context"
55 "encoding/json"
66 "fmt"
7+ "sort"
78 "strconv"
89 "strings"
910 "sync"
@@ -43,6 +44,7 @@ type Mysis struct {
4344 lastServerTick int64
4445 lastServerTickAt time.Time
4546 tickDuration time.Duration
47+ nudgeCh chan struct{}
4648 }
4749
4850 type contextStats struct {
@@ -66,6 +68,7 @@ func NewMysis(id, name string, createdAt time.Time, p provider.Provider, s *stor
6668 bus: bus,
6769 state: MysisStateIdle,
6870 activityState: ActivityStateIdle,
71+ nudgeCh: make(chan struct{}, 1),
6972 }
7073 }
7174
@@ -212,7 +215,7 @@ func (a *Mysis) Stop() error {
212215 a.mu.Lock()
213216 if a.state != MysisStateRunning {
214217 a.mu.Unlock()
215- return fmt.Errorf("mysis not running")
218+ return nil
216219 }
217220
218221 if a.cancel != nil {
@@ -225,6 +228,10 @@ func (a *Mysis) Stop() error {
225228 defer a.turnMu.Unlock()
226229
227230 a.mu.Lock()
231+ if a.state != MysisStateRunning {
232+ a.mu.Unlock()
233+ return nil
234+ }
228235 a.cancel = nil
229236 a.ctx = nil
230237
@@ -298,7 +305,7 @@ func (a *Mysis) SendMessageFrom(content string, source store.MemorySource, sende
298305 mcpTools, err := mcpProxy.ListTools(ctx)
299306 if err != nil {
300307 // Log error but continue - mysis can still chat without tools
301- a.bus.Publish(Event{
308+ a.publishCriticalEvent(Event{
302309 Type: EventMysisError,
303310 MysisID: a.id,
304311 MysisName: a.name,
@@ -328,7 +335,7 @@ func (a *Mysis) SendMessageFrom(content string, source store.MemorySource, sende
328335 }
329336 }
330337 } else {
331- a.bus.Publish(Event{
338+ a.publishCriticalEvent(Event{
332339 Type: EventMysisError,
333340 MysisID: a.id,
334341 MysisName: a.name,
@@ -693,7 +700,7 @@ func (a *Mysis) setError(err error) {
693700 a.lastError = err
694701 a.mu.Unlock()
695702
696- a.bus.Publish(Event{
703+ a.publishCriticalEvent(Event{
697704 Type: EventMysisError,
698705 MysisID: a.id,
699706 MysisName: a.name,
@@ -845,12 +852,17 @@ func (a *Mysis) run(ctx context.Context) {
845852 a.mu.RUnlock()
846853
847854 if isRunning && a.shouldNudge(time.Now()) {
848- // Only nudge if not already in a turn
849- if a.turnMu.TryLock() {
850- a.turnMu.Unlock()
851- go a.SendMessage(a.buildContinuePrompt(), store.MemorySourceSystem)
855+ select {
856+ case a.nudgeCh <- struct{}{}:
857+ default:
852858 }
853859 }
860+ case <-a.nudgeCh:
861+ if !a.turnMu.TryLock() {
862+ continue
863+ }
864+ a.turnMu.Unlock()
865+ go a.SendMessage(a.buildContinuePrompt(), store.MemorySourceSystem)
854866 }
855867 }
856868 }
@@ -964,7 +976,7 @@ func (a *Mysis) updateActivityFromToolResult(toolName string, result *mcp.ToolRe
964976 var currentTick int64
965977 var currentTickOK bool
966978 if ok {
967- currentTick, currentTickOK = findIntField(payload, "current_tick", "tick")
979+ currentTick, currentTickOK = findCurrentTick(payload)
968980 if currentTickOK {
969981 a.updateServerTick(now, currentTick)
970982 }
@@ -1089,23 +1101,69 @@ func parseToolResultPayload(result *mcp.ToolResult) (interface{}, bool) {
10891101 }
10901102
10911103 func findIntField(payload interface{}, keys ...string) (int64, bool) {
1092- switch value := payload.(type) {
1093- case map[string]interface{}:
1094- for _, key := range keys {
1095- if raw, ok := value[key]; ok {
1096- if number, ok := normalizeInt(raw); ok {
1097- return number, true
1104+ queue := []interface{}{payload}
1105+
1106+ for len(queue) > 0 {
1107+ current := queue[0]
1108+ queue = queue[1:]
1109+
1110+ switch value := current.(type) {
1111+ case map[string]interface{}:
1112+ for _, key := range keys {
1113+ if raw, ok := value[key]; ok {
1114+ if number, ok := normalizeInt(raw); ok {
1115+ return number, true
1116+ }
10981117 }
10991118 }
1100- }
1101- for _, child := range value {
1102- if number, ok := findIntField(child, keys...); ok {
1103- return number, true
1119+
1120+ childKeys := make([]string, 0, len(value))
1121+ for key := range value {
1122+ childKeys = append(childKeys, key)
1123+ }
1124+ sort.Strings(childKeys)
1125+ for _, key := range childKeys {
1126+ queue = append(queue, value[key])
11041127 }
1128+ case []interface{}:
1129+ queue = append(queue, value...)
11051130 }
1106- case []interface{}:
1107- for _, child := range value {
1108- if number, ok := findIntField(child, keys...); ok {
1131+ }
1132+
1133+ return 0, false
1134+}
1135+
1136+func findCurrentTick(payload interface{}) (int64, bool) {
1137+ if number, ok := findIntFieldAtKeys(payload, "current_tick"); ok {
1138+ return number, true
1139+ }
1140+ if number, ok := findIntFieldInWrappers(payload, "current_tick"); ok {
1141+ return number, true
1142+ }
1143+ if number, ok := findIntField(payload, "current_tick"); ok {
1144+ return number, true
1145+ }
1146+ if number, ok := findIntFieldAtKeys(payload, "tick"); ok {
1147+ return number, true
1148+ }
1149+ if number, ok := findIntFieldInWrappers(payload, "tick"); ok {
1150+ return number, true
1151+ }
1152+ if number, ok := findIntField(payload, "tick"); ok {
1153+ return number, true
1154+ }
1155+ return 0, false
1156+}
1157+
1158+func findIntFieldAtKeys(payload interface{}, keys ...string) (int64, bool) {
1159+ value, ok := payload.(map[string]interface{})
1160+ if !ok {
1161+ return 0, false
1162+ }
1163+
1164+ for _, key := range keys {
1165+ if raw, ok := value[key]; ok {
1166+ if number, ok := normalizeInt(raw); ok {
11091167 return number, true
11101168 }
11111169 }
@@ -1114,6 +1172,26 @@ func findIntField(payload interface{}, keys ...string) (int64, bool) {
11141172 return 0, false
11151173 }
11161174
1175+func findIntFieldInWrappers(payload interface{}, key string) (int64, bool) {
1176+ value, ok := payload.(map[string]interface{})
1177+ if !ok {
1178+ return 0, false
1179+ }
1180+
1181+ wrappers := []string{"data", "result", "payload", "response"}
1182+ for _, wrapper := range wrappers {
1183+ child, ok := value[wrapper]
1184+ if !ok {
1185+ continue
1186+ }
1187+ if number, ok := findIntField(child, key); ok {
1188+ return number, true
1189+ }
1190+ }
1191+
1192+ return 0, false
1193+}
1194+
11171195 func normalizeInt(value interface{}) (int64, bool) {
11181196 switch v := value.(type) {
11191197 case int:
@@ -1140,7 +1218,7 @@ func normalizeInt(value interface{}) (int64, bool) {
11401218 }
11411219
11421220 func (a *Mysis) emitStateChange(oldState, newState MysisState) {
1143- a.bus.Publish(Event{
1221+ a.publishCriticalEvent(Event{
11441222 Type: EventMysisStateChanged,
11451223 MysisID: a.id,
11461224 MysisName: a.name,
@@ -1159,10 +1237,16 @@ func (a *Mysis) setErrorState(err error) {
11591237 a.lastError = err
11601238 a.mu.Unlock()
11611239
1162- a.store.UpdateMysisState(a.id, store.MysisStateErrored)
1240+ if err := a.store.UpdateMysisState(a.id, store.MysisStateErrored); err != nil {
1241+ log.Warn().
1242+ Err(err).
1243+ Str("mysis_id", a.id).
1244+ Str("mysis_name", a.name).
1245+ Msg("failed to update mysis state to errored")
1246+ }
11631247 a.emitStateChange(oldState, MysisStateErrored)
11641248
1165- a.bus.Publish(Event{
1249+ a.publishCriticalEvent(Event{
11661250 Type: EventMysisError,
11671251 MysisID: a.id,
11681252 MysisName: a.name,
@@ -1170,3 +1254,15 @@ func (a *Mysis) setErrorState(err error) {
11701254 Timestamp: time.Now(),
11711255 })
11721256 }
1257+
1258+func (a *Mysis) publishCriticalEvent(event Event) {
1259+ if a.bus.PublishBlocking(event, constants.EventBusPublishTimeout) {
1260+ return
1261+ }
1262+
1263+ log.Warn().
1264+ Str("event_type", string(event.Type)).
1265+ Str("mysis_id", a.id).
1266+ Str("mysis_name", a.name).
1267+ Msg("event bus publish timeout")
1268+}
internal/core/mysis_test.go+86−3
@@ -79,9 +79,46 @@ func TestMysisLifecycle(t *testing.T) {
7979 t.Errorf("expected state=stopped, got %s", mysis.State())
8080 }
8181
82- // Stop again should error
83- if err := mysis.Stop(); err == nil {
84- t.Error("expected error stopping already stopped mysis")
82+ // Stop again should be idempotent
83+ if err := mysis.Stop(); err != nil {
84+ t.Errorf("expected no error stopping already stopped mysis, got %v", err)
85+ }
86+}
87+
88+func TestMysisConcurrentStopDuringTurn(t *testing.T) {
89+ s, bus, cleanup := setupMysisTest(t)
90+ defer cleanup()
91+
92+ stored, err := s.CreateMysis("concurrent", "mock", "test-model", 0.7)
93+ if err != nil {
94+ t.Fatalf("CreateMysis() error: %v", err)
95+ }
96+
97+ mock := provider.NewMock("mock", "ok").SetDelay(50 * time.Millisecond)
98+ mysis := NewMysis(stored.ID, stored.Name, stored.CreatedAt, mock, s, bus)
99+
100+ if err := mysis.Start(); err != nil {
101+ t.Fatalf("Start() error: %v", err)
102+ }
103+
104+ go func() {
105+ _ = mysis.SendMessage("ping", store.MemorySourceDirect)
106+ }()
107+
108+ time.Sleep(10 * time.Millisecond)
109+
110+ done := make(chan error, 1)
111+ go func() {
112+ done <- mysis.Stop()
113+ }()
114+
115+ select {
116+ case err := <-done:
117+ if err != nil {
118+ t.Fatalf("Stop() error: %v", err)
119+ }
120+ case <-time.After(500 * time.Millisecond):
121+ t.Fatal("timeout waiting for Stop")
85122 }
86123 }
87124
@@ -818,3 +855,49 @@ func TestMysisActivityTravelArrivalTickReached(t *testing.T) {
818855 t.Fatal("expected activityUntil to be zero when arrival tick reached")
819856 }
820857 }
858+
859+func TestFindCurrentTick(t *testing.T) {
860+ tests := []struct {
861+ name string
862+ payload interface{}
863+ want int64
864+ ok bool
865+ }{
866+ {
867+ name: "top_level_current_tick",
868+ payload: map[string]interface{}{"current_tick": int64(42)},
869+ want: 42,
870+ ok: true,
871+ },
872+ {
873+ name: "wrapper_current_tick",
874+ payload: map[string]interface{}{"data": map[string]interface{}{"current_tick": int64(88)}},
875+ want: 88,
876+ ok: true,
877+ },
878+ {
879+ name: "wrapper_tick_fallback",
880+ payload: map[string]interface{}{"result": map[string]interface{}{"tick": int64(9)}},
881+ want: 9,
882+ ok: true,
883+ },
884+ {
885+ name: "missing_tick",
886+ payload: map[string]interface{}{"status": "ok"},
887+ want: 0,
888+ ok: false,
889+ },
890+ }
891+
892+ for _, test := range tests {
893+ t.Run(test.name, func(t *testing.T) {
894+ got, ok := findCurrentTick(test.payload)
895+ if ok != test.ok {
896+ t.Fatalf("expected ok=%v, got %v", test.ok, ok)
897+ }
898+ if got != test.want {
899+ t.Fatalf("expected tick %d, got %d", test.want, got)
900+ }
901+ })
902+ }
903+}