6312
Ga
go-opencode-ai-zen-sdk / files
go-opencode-ai-zen-sdk / cmd / test_models / main.go
9.1 KB
1package main 2 3import ( 4 "context" 5 "encoding/json" 6 "fmt" 7 "os" 8 "strings" 9 "time" 10 11 zen "github.com/sacenox/go-opencode-ai-zen-sdk" 12) 13 14// One representative model per endpoint type. 15var probeModels = []string{"gpt-5.1", "claude-sonnet-4-6", "gemini-3-flash", "kimi-k2-thinking"} 16 17type testResult struct { 18 Model string 19 Endpoint string 20 Success bool 21 Error string 22 Latency time.Duration 23 Request string 24 Response string 25 Stream bool 26 InputTokens int 27 OutputTokens int 28} 29 30func main() { 31 apiKey := os.Getenv("OPENCODE_API_KEY") 32 if apiKey == "" { 33 fmt.Fprintln(os.Stderr, "OPENCODE_API_KEY environment variable is required") 34 os.Exit(1) 35 } 36 37 client, err := zen.NewClient(zen.Config{APIKey: apiKey}) 38 if err != nil { 39 fmt.Fprintf(os.Stderr, "Failed to create client: %v\n", err) 40 os.Exit(1) 41 } 42 43 ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) 44 defer cancel() 45 46 sections := []struct { 47 name string 48 fn func(context.Context, *zen.Client, string) testResult 49 }{ 50 {"StreamEvents", testStreamEvents}, 51 {"StreamParsed", testStreamParsed}, 52 {"ToolHistory (stream)", testToolHistoryStream}, 53 {"Reasoning (stream)", testReasoningStream}, 54 } 55 56 var results []testResult 57 58 for _, s := range sections { 59 fmt.Printf("=== %s ===\n", s.name) 60 for _, modelID := range probeModels { 61 r := s.fn(ctx, client, modelID) 62 results = append(results, r) 63 printResult(r) 64 } 65 fmt.Println() 66 } 67 68 successCount := 0 69 for _, r := range results { 70 if r.Success { 71 successCount++ 72 } 73 } 74 fmt.Printf("=== Summary: %d/%d passed ===\n", successCount, len(results)) 75 for _, r := range results { 76 if !r.Success { 77 fmt.Printf(" FAILED [stream] %s (%s): %s\n", r.Model, r.Endpoint, r.Error) 78 } 79 } 80} 81 82func testStreamEvents(ctx context.Context, client *zen.Client, modelID string) testResult { 83 start := time.Now() 84 endpoint := routeForModel(modelID) 85 req := zen.NormalizedRequest{ 86 Model: modelID, 87 Messages: []zen.NormalizedMessage{{Role: "user", Content: "Say ok"}}, 88 } 89 reqBody, _ := json.Marshal(req) 90 91 eventCh, errCh, err := client.StreamEvents(ctx, req) 92 return drainUnifiedStream(modelID, endpoint, string(reqBody), eventCh, errCh, err, start) 93} 94 95func testStreamParsed(ctx context.Context, client *zen.Client, modelID string) testResult { 96 start := time.Now() 97 endpoint := routeForModel(modelID) 98 req := zen.NormalizedRequest{ 99 Model: modelID, 100 Messages: []zen.NormalizedMessage{{Role: "user", Content: "Say ok"}}, 101 } 102 reqBody, _ := json.Marshal(req) 103 104 deltaCh, errCh, err := client.Stream(ctx, req) 105 if err != nil { 106 return makeResult(modelID, endpoint, true, string(reqBody), "", err, time.Since(start), 0, 0) 107 } 108 109 var textBuf, reasoningBuf strings.Builder 110 var count, inTok, outTok int 111 for d := range deltaCh { 112 count++ 113 switch d.Type { 114 case zen.DeltaText: 115 textBuf.WriteString(d.Content) 116 case zen.DeltaReasoning: 117 reasoningBuf.WriteString(d.Content) 118 case zen.DeltaUsage: 119 if d.InputTokens > inTok { 120 inTok = d.InputTokens 121 } 122 outTok += d.OutputTokens 123 } 124 } 125 if streamErr := <-errCh; streamErr != nil { 126 return makeResult(modelID, endpoint, true, string(reqBody), "", streamErr, time.Since(start), 0, 0) 127 } 128 129 resp := fmt.Sprintf("deltas=%d text=%s reasoning=%s", count, truncate(textBuf.String(), 40), truncate(reasoningBuf.String(), 40)) 130 return makeResult(modelID, endpoint, true, string(reqBody), resp, nil, time.Since(start), inTok, outTok) 131} 132 133// toolHistory returns a pre-built two-turn tool-use conversation: 134// 135// user → assistant+tool_call → tool_result → user follow-up 136func toolHistory() []zen.NormalizedMessage { 137 return []zen.NormalizedMessage{ 138 {Role: "user", Content: "What is 3 + 4?"}, 139 { 140 Role: "assistant", 141 ToolCalls: []zen.NormalizedToolCall{ 142 {ID: "call_1", Name: "add", Arguments: json.RawMessage(`{"a":3,"b":4}`)}, 143 }, 144 }, 145 {Role: "tool", Content: "7", ToolCallID: "call_1"}, 146 {Role: "user", Content: "Thanks, what is the result doubled?"}, 147 } 148} 149 150func testToolHistoryStream(ctx context.Context, client *zen.Client, modelID string) testResult { 151 start := time.Now() 152 endpoint := routeForModel(modelID) 153 req := zen.NormalizedRequest{ 154 Model: modelID, 155 System: "You are a calculator assistant. When given a tool result, use it to answer the user.", 156 Messages: toolHistory(), 157 Tools: []zen.NormalizedTool{ 158 { 159 Name: "add", 160 Description: "Adds two numbers", 161 Parameters: json.RawMessage(`{"type":"object","properties":{"a":{"type":"number"},"b":{"type":"number"}},"required":["a","b"]}`), 162 }, 163 }, 164 } 165 reqBody, _ := json.Marshal(req) 166 167 deltaCh, errCh, err := client.Stream(ctx, req) 168 if err != nil { 169 return makeResult(modelID, endpoint, true, string(reqBody), "", err, time.Since(start), 0, 0) 170 } 171 172 var textBuf strings.Builder 173 var count, inTok, outTok int 174 for d := range deltaCh { 175 count++ 176 switch d.Type { 177 case zen.DeltaText: 178 textBuf.WriteString(d.Content) 179 case zen.DeltaUsage: 180 if d.InputTokens > inTok { 181 inTok = d.InputTokens 182 } 183 outTok += d.OutputTokens 184 } 185 } 186 if streamErr := <-errCh; streamErr != nil { 187 return makeResult(modelID, endpoint, true, string(reqBody), "", streamErr, time.Since(start), 0, 0) 188 } 189 190 resp := fmt.Sprintf("deltas=%d text=%s", count, truncate(textBuf.String(), 60)) 191 return makeResult(modelID, endpoint, true, string(reqBody), resp, nil, time.Since(start), inTok, outTok) 192} 193 194func testReasoningStream(ctx context.Context, client *zen.Client, modelID string) testResult { 195 start := time.Now() 196 endpoint := routeForModel(modelID) 197 req := zen.NormalizedRequest{ 198 Model: modelID, 199 Messages: []zen.NormalizedMessage{{Role: "user", Content: "What is 2 + 2? Think step by step."}}, 200 Reasoning: &zen.NormalizedReasoning{Effort: "low"}, 201 } 202 reqBody, _ := json.Marshal(req) 203 204 deltaCh, errCh, err := client.Stream(ctx, req) 205 if err != nil { 206 return makeResult(modelID, endpoint, true, string(reqBody), "", err, time.Since(start), 0, 0) 207 } 208 209 var textBuf, reasoningBuf strings.Builder 210 var deltaCount, inTok, outTok int 211 for d := range deltaCh { 212 deltaCount++ 213 switch d.Type { 214 case zen.DeltaText: 215 textBuf.WriteString(d.Content) 216 case zen.DeltaReasoning: 217 reasoningBuf.WriteString(d.Content) 218 case zen.DeltaUsage: 219 if d.InputTokens > inTok { 220 inTok = d.InputTokens 221 } 222 outTok += d.OutputTokens 223 } 224 } 225 if streamErr := <-errCh; streamErr != nil { 226 return makeResult(modelID, endpoint, true, string(reqBody), "", streamErr, time.Since(start), 0, 0) 227 } 228 229 if reasoningBuf.Len() == 0 { 230 return makeResult(modelID, endpoint, true, string(reqBody), 231 fmt.Sprintf("deltas=%d text=%s", deltaCount, truncate(textBuf.String(), 60)), 232 fmt.Errorf("model %s: stream contained no reasoning/thinking deltas", modelID), 233 time.Since(start), inTok, outTok) 234 } 235 return makeResult(modelID, endpoint, true, string(reqBody), 236 fmt.Sprintf("deltas=%d reasoning=%s text=%s", 237 deltaCount, 238 truncate(reasoningBuf.String(), 40), 239 truncate(textBuf.String(), 40)), 240 nil, time.Since(start), inTok, outTok) 241} 242 243// drainUnifiedStream consumes a UnifiedEvent channel and builds a testResult. 244// StreamEvents operates at the raw event level and does not parse DeltaUsage, 245// so token counts are not available here. 246func drainUnifiedStream(modelID, endpoint, reqBody string, eventCh <-chan zen.UnifiedEvent, errCh <-chan error, initErr error, start time.Time) testResult { 247 if initErr != nil { 248 return makeResult(modelID, endpoint, true, reqBody, "", initErr, time.Since(start), 0, 0) 249 } 250 var count int 251 var last string 252 var resolved string 253 for ev := range eventCh { 254 count++ 255 resolved = string(ev.Endpoint) 256 if len(ev.Data) > 0 { 257 last = string(ev.Data) 258 } 259 } 260 if err := <-errCh; err != nil { 261 return makeResult(modelID, endpoint, true, reqBody, "", err, time.Since(start), 0, 0) 262 } 263 if resolved != "" { 264 endpoint = resolved 265 } 266 return makeResult(modelID, endpoint, true, reqBody, fmt.Sprintf("events=%d last=%s", count, truncate(last, 80)), nil, time.Since(start), 0, 0) 267} 268 269func makeResult(modelID, endpoint string, stream bool, req, resp string, err error, latency time.Duration, inTok, outTok int) testResult { 270 r := testResult{ 271 Model: modelID, 272 Endpoint: endpoint, 273 Stream: stream, 274 Latency: latency, 275 Request: req, 276 Response: resp, 277 InputTokens: inTok, 278 OutputTokens: outTok, 279 } 280 if err != nil { 281 r.Error = err.Error() 282 } else { 283 r.Success = true 284 } 285 return r 286} 287 288func routeForModel(model string) string { 289 m := strings.ToLower(strings.TrimSpace(model)) 290 if strings.HasPrefix(m, "opencode/") { 291 m = strings.TrimPrefix(m, "opencode/") 292 } 293 switch { 294 case strings.HasPrefix(m, "gpt-"): 295 return "responses" 296 case strings.HasPrefix(m, "claude-"): 297 return "messages" 298 case strings.HasPrefix(m, "gemini-"): 299 return "models" 300 default: 301 return "chat_completions" 302 } 303} 304 305func printResult(r testResult) { 306 status := "✓" 307 if !r.Success { 308 status = "✗" 309 } 310 mode := "stream" 311 usage := "-" 312 if r.InputTokens > 0 || r.OutputTokens > 0 { 313 usage = fmt.Sprintf("in=%d out=%d", r.InputTokens, r.OutputTokens) 314 } 315 fmt.Printf(" %s %-25s [%-15s] [%-10s] %-20s %v\n", status, r.Model, r.Endpoint, mode, usage, r.Latency) 316} 317 318func truncate(s string, max int) string { 319 if len(s) <= max { 320 return s 321 } 322 return s[:max] + "..." 323}