xonecas.io
6312
Ga

go-opencode-ai-zen-sdk / files

go-opencode-ai-zen-sdk / cmd / test_models / main.go

9.1 KB
1package main
2
3import (
4	"context"
5	"encoding/json"
6	"fmt"
7	"os"
8	"strings"
9	"time"
10
11	zen "github.com/sacenox/go-opencode-ai-zen-sdk"
12)
13
14// One representative model per endpoint type.
15var probeModels = []string{"gpt-5.1", "claude-sonnet-4-6", "gemini-3-flash", "kimi-k2-thinking"}
16
17type testResult struct {
18	Model        string
19	Endpoint     string
20	Success      bool
21	Error        string
22	Latency      time.Duration
23	Request      string
24	Response     string
25	Stream       bool
26	InputTokens  int
27	OutputTokens int
28}
29
30func main() {
31	apiKey := os.Getenv("OPENCODE_API_KEY")
32	if apiKey == "" {
33		fmt.Fprintln(os.Stderr, "OPENCODE_API_KEY environment variable is required")
34		os.Exit(1)
35	}
36
37	client, err := zen.NewClient(zen.Config{APIKey: apiKey})
38	if err != nil {
39		fmt.Fprintf(os.Stderr, "Failed to create client: %v\n", err)
40		os.Exit(1)
41	}
42
43	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
44	defer cancel()
45
46	sections := []struct {
47		name string
48		fn   func(context.Context, *zen.Client, string) testResult
49	}{
50		{"StreamEvents", testStreamEvents},
51		{"StreamParsed", testStreamParsed},
52		{"ToolHistory (stream)", testToolHistoryStream},
53		{"Reasoning (stream)", testReasoningStream},
54	}
55
56	var results []testResult
57
58	for _, s := range sections {
59		fmt.Printf("=== %s ===\n", s.name)
60		for _, modelID := range probeModels {
61			r := s.fn(ctx, client, modelID)
62			results = append(results, r)
63			printResult(r)
64		}
65		fmt.Println()
66	}
67
68	successCount := 0
69	for _, r := range results {
70		if r.Success {
71			successCount++
72		}
73	}
74	fmt.Printf("=== Summary: %d/%d passed ===\n", successCount, len(results))
75	for _, r := range results {
76		if !r.Success {
77			fmt.Printf("  FAILED [stream] %s (%s): %s\n", r.Model, r.Endpoint, r.Error)
78		}
79	}
80}
81
82func testStreamEvents(ctx context.Context, client *zen.Client, modelID string) testResult {
83	start := time.Now()
84	endpoint := routeForModel(modelID)
85	req := zen.NormalizedRequest{
86		Model:    modelID,
87		Messages: []zen.NormalizedMessage{{Role: "user", Content: "Say ok"}},
88	}
89	reqBody, _ := json.Marshal(req)
90
91	eventCh, errCh, err := client.StreamEvents(ctx, req)
92	return drainUnifiedStream(modelID, endpoint, string(reqBody), eventCh, errCh, err, start)
93}
94
95func testStreamParsed(ctx context.Context, client *zen.Client, modelID string) testResult {
96	start := time.Now()
97	endpoint := routeForModel(modelID)
98	req := zen.NormalizedRequest{
99		Model:    modelID,
100		Messages: []zen.NormalizedMessage{{Role: "user", Content: "Say ok"}},
101	}
102	reqBody, _ := json.Marshal(req)
103
104	deltaCh, errCh, err := client.Stream(ctx, req)
105	if err != nil {
106		return makeResult(modelID, endpoint, true, string(reqBody), "", err, time.Since(start), 0, 0)
107	}
108
109	var textBuf, reasoningBuf strings.Builder
110	var count, inTok, outTok int
111	for d := range deltaCh {
112		count++
113		switch d.Type {
114		case zen.DeltaText:
115			textBuf.WriteString(d.Content)
116		case zen.DeltaReasoning:
117			reasoningBuf.WriteString(d.Content)
118		case zen.DeltaUsage:
119			if d.InputTokens > inTok {
120				inTok = d.InputTokens
121			}
122			outTok += d.OutputTokens
123		}
124	}
125	if streamErr := <-errCh; streamErr != nil {
126		return makeResult(modelID, endpoint, true, string(reqBody), "", streamErr, time.Since(start), 0, 0)
127	}
128
129	resp := fmt.Sprintf("deltas=%d text=%s reasoning=%s", count, truncate(textBuf.String(), 40), truncate(reasoningBuf.String(), 40))
130	return makeResult(modelID, endpoint, true, string(reqBody), resp, nil, time.Since(start), inTok, outTok)
131}
132
133// toolHistory returns a pre-built two-turn tool-use conversation:
134//
135//	user → assistant+tool_call → tool_result → user follow-up
136func toolHistory() []zen.NormalizedMessage {
137	return []zen.NormalizedMessage{
138		{Role: "user", Content: "What is 3 + 4?"},
139		{
140			Role: "assistant",
141			ToolCalls: []zen.NormalizedToolCall{
142				{ID: "call_1", Name: "add", Arguments: json.RawMessage(`{"a":3,"b":4}`)},
143			},
144		},
145		{Role: "tool", Content: "7", ToolCallID: "call_1"},
146		{Role: "user", Content: "Thanks, what is the result doubled?"},
147	}
148}
149
150func testToolHistoryStream(ctx context.Context, client *zen.Client, modelID string) testResult {
151	start := time.Now()
152	endpoint := routeForModel(modelID)
153	req := zen.NormalizedRequest{
154		Model:    modelID,
155		System:   "You are a calculator assistant. When given a tool result, use it to answer the user.",
156		Messages: toolHistory(),
157		Tools: []zen.NormalizedTool{
158			{
159				Name:        "add",
160				Description: "Adds two numbers",
161				Parameters:  json.RawMessage(`{"type":"object","properties":{"a":{"type":"number"},"b":{"type":"number"}},"required":["a","b"]}`),
162			},
163		},
164	}
165	reqBody, _ := json.Marshal(req)
166
167	deltaCh, errCh, err := client.Stream(ctx, req)
168	if err != nil {
169		return makeResult(modelID, endpoint, true, string(reqBody), "", err, time.Since(start), 0, 0)
170	}
171
172	var textBuf strings.Builder
173	var count, inTok, outTok int
174	for d := range deltaCh {
175		count++
176		switch d.Type {
177		case zen.DeltaText:
178			textBuf.WriteString(d.Content)
179		case zen.DeltaUsage:
180			if d.InputTokens > inTok {
181				inTok = d.InputTokens
182			}
183			outTok += d.OutputTokens
184		}
185	}
186	if streamErr := <-errCh; streamErr != nil {
187		return makeResult(modelID, endpoint, true, string(reqBody), "", streamErr, time.Since(start), 0, 0)
188	}
189
190	resp := fmt.Sprintf("deltas=%d text=%s", count, truncate(textBuf.String(), 60))
191	return makeResult(modelID, endpoint, true, string(reqBody), resp, nil, time.Since(start), inTok, outTok)
192}
193
194func testReasoningStream(ctx context.Context, client *zen.Client, modelID string) testResult {
195	start := time.Now()
196	endpoint := routeForModel(modelID)
197	req := zen.NormalizedRequest{
198		Model:     modelID,
199		Messages:  []zen.NormalizedMessage{{Role: "user", Content: "What is 2 + 2? Think step by step."}},
200		Reasoning: &zen.NormalizedReasoning{Effort: "low"},
201	}
202	reqBody, _ := json.Marshal(req)
203
204	deltaCh, errCh, err := client.Stream(ctx, req)
205	if err != nil {
206		return makeResult(modelID, endpoint, true, string(reqBody), "", err, time.Since(start), 0, 0)
207	}
208
209	var textBuf, reasoningBuf strings.Builder
210	var deltaCount, inTok, outTok int
211	for d := range deltaCh {
212		deltaCount++
213		switch d.Type {
214		case zen.DeltaText:
215			textBuf.WriteString(d.Content)
216		case zen.DeltaReasoning:
217			reasoningBuf.WriteString(d.Content)
218		case zen.DeltaUsage:
219			if d.InputTokens > inTok {
220				inTok = d.InputTokens
221			}
222			outTok += d.OutputTokens
223		}
224	}
225	if streamErr := <-errCh; streamErr != nil {
226		return makeResult(modelID, endpoint, true, string(reqBody), "", streamErr, time.Since(start), 0, 0)
227	}
228
229	if reasoningBuf.Len() == 0 {
230		return makeResult(modelID, endpoint, true, string(reqBody),
231			fmt.Sprintf("deltas=%d text=%s", deltaCount, truncate(textBuf.String(), 60)),
232			fmt.Errorf("model %s: stream contained no reasoning/thinking deltas", modelID),
233			time.Since(start), inTok, outTok)
234	}
235	return makeResult(modelID, endpoint, true, string(reqBody),
236		fmt.Sprintf("deltas=%d reasoning=%s text=%s",
237			deltaCount,
238			truncate(reasoningBuf.String(), 40),
239			truncate(textBuf.String(), 40)),
240		nil, time.Since(start), inTok, outTok)
241}
242
243// drainUnifiedStream consumes a UnifiedEvent channel and builds a testResult.
244// StreamEvents operates at the raw event level and does not parse DeltaUsage,
245// so token counts are not available here.
246func drainUnifiedStream(modelID, endpoint, reqBody string, eventCh <-chan zen.UnifiedEvent, errCh <-chan error, initErr error, start time.Time) testResult {
247	if initErr != nil {
248		return makeResult(modelID, endpoint, true, reqBody, "", initErr, time.Since(start), 0, 0)
249	}
250	var count int
251	var last string
252	var resolved string
253	for ev := range eventCh {
254		count++
255		resolved = string(ev.Endpoint)
256		if len(ev.Data) > 0 {
257			last = string(ev.Data)
258		}
259	}
260	if err := <-errCh; err != nil {
261		return makeResult(modelID, endpoint, true, reqBody, "", err, time.Since(start), 0, 0)
262	}
263	if resolved != "" {
264		endpoint = resolved
265	}
266	return makeResult(modelID, endpoint, true, reqBody, fmt.Sprintf("events=%d last=%s", count, truncate(last, 80)), nil, time.Since(start), 0, 0)
267}
268
269func makeResult(modelID, endpoint string, stream bool, req, resp string, err error, latency time.Duration, inTok, outTok int) testResult {
270	r := testResult{
271		Model:        modelID,
272		Endpoint:     endpoint,
273		Stream:       stream,
274		Latency:      latency,
275		Request:      req,
276		Response:     resp,
277		InputTokens:  inTok,
278		OutputTokens: outTok,
279	}
280	if err != nil {
281		r.Error = err.Error()
282	} else {
283		r.Success = true
284	}
285	return r
286}
287
288func routeForModel(model string) string {
289	m := strings.ToLower(strings.TrimSpace(model))
290	if strings.HasPrefix(m, "opencode/") {
291		m = strings.TrimPrefix(m, "opencode/")
292	}
293	switch {
294	case strings.HasPrefix(m, "gpt-"):
295		return "responses"
296	case strings.HasPrefix(m, "claude-"):
297		return "messages"
298	case strings.HasPrefix(m, "gemini-"):
299		return "models"
300	default:
301		return "chat_completions"
302	}
303}
304
305func printResult(r testResult) {
306	status := "✓"
307	if !r.Success {
308		status = "✗"
309	}
310	mode := "stream"
311	usage := "-"
312	if r.InputTokens > 0 || r.OutputTokens > 0 {
313		usage = fmt.Sprintf("in=%d out=%d", r.InputTokens, r.OutputTokens)
314	}
315	fmt.Printf("  %s %-25s [%-15s] [%-10s] %-20s %v\n", status, r.Model, r.Endpoint, mode, usage, r.Latency)
316}
317
318func truncate(s string, max int) string {
319	if len(s) <= max {
320		return s
321	}
322	return s[:max] + "..."
323}