xonecas.io
62159
Sy

symb / files

symb / internal / provider / ollama.go

12.5 KB
1package provider
2
3import (
4	"bufio"
5	"bytes"
6	"context"
7	"encoding/json"
8	"errors"
9	"fmt"
10	"io"
11	"net/http"
12	"strings"
13	"time"
14
15	"github.com/rs/zerolog/log"
16)
17
18const roleSystem = "system"
19
20type OllamaProvider struct {
21	name        string
22	baseURL     string
23	httpClient  *http.Client
24	model       string
25	temperature float64
26}
27
28func NewOllama(endpoint, model string) *OllamaProvider {
29	return NewOllamaWithTemp("ollama", endpoint, model, 0.7)
30}
31
32func NewOllamaWithTemp(name string, endpoint, model string, temperature float64) *OllamaProvider {
33	baseURL := strings.TrimRight(endpoint, "/") + "/v1"
34
35	return &OllamaProvider{
36		name:        name,
37		baseURL:     baseURL,
38		httpClient:  &http.Client{},
39		model:       model,
40		temperature: temperature,
41	}
42}
43
44func (p *OllamaProvider) Name() string {
45	return p.name
46}
47
48func (p *OllamaProvider) ChatStream(ctx context.Context, messages []Message, tools []Tool) (<-chan StreamEvent, error) {
49	req := ollamaChatRequest{
50		Model:         p.model,
51		Messages:      mergeConsecutiveSystemMessages(toOllamaMessages(messages)),
52		Tools:         toOllamaTools(tools),
53		Temperature:   float32(p.temperature),
54		Stream:        true,
55		StreamOptions: &chatStreamOptions{IncludeUsage: true},
56	}
57	body, err := json.Marshal(req)
58	if err != nil {
59		return nil, err
60	}
61
62	reader, err := httpDoSSE(ctx, httpRequestConfig{
63		client:   p.httpClient,
64		url:      p.baseURL + "/chat/completions",
65		body:     body,
66		provider: p.name,
67		model:    p.model,
68	})
69	if err != nil {
70		return nil, err
71	}
72
73	ch := make(chan StreamEvent)
74	go func() {
75		defer close(ch)
76		defer reader.Close()
77		parseSSEStream(ctx, reader, ch)
78	}()
79
80	return ch, nil
81}
82
83func (p *OllamaProvider) ListModels(ctx context.Context) ([]Model, error) {
84	baseURL := strings.TrimRight(p.baseURL, "/v1")
85	url := baseURL + "/api/tags"
86
87	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
88	if err != nil {
89		return nil, err
90	}
91	req.Header.Set("Accept", "application/json")
92
93	resp, err := p.httpClient.Do(req)
94	if err != nil {
95		return nil, err
96	}
97	defer resp.Body.Close()
98
99	if resp.StatusCode != http.StatusOK {
100		body, _ := io.ReadAll(resp.Body)
101		return nil, fmt.Errorf("list models status %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
102	}
103
104	var listResp ollamaListResponse
105	if err := json.NewDecoder(resp.Body).Decode(&listResp); err != nil {
106		return nil, err
107	}
108
109	models := make([]Model, len(listResp.Models))
110	for i, m := range listResp.Models {
111		models[i] = Model{
112			Name:       m.Name,
113			Size:       m.Size,
114			Digest:     m.Digest,
115			ModifiedAt: m.ModifiedAt,
116			Format:     m.Details.Format,
117			Family:     m.Details.Family,
118			ParamSize:  m.Details.ParamSize,
119			QuantLevel: m.Details.QuantLevel,
120		}
121	}
122	return models, nil
123}
124
125func (p *OllamaProvider) Close() error {
126	if p.httpClient != nil {
127		p.httpClient.CloseIdleConnections()
128	}
129	return nil
130}
131
132type ollamaListResponse struct {
133	Models []ollamaModel `json:"models"`
134}
135
136type ollamaModel struct {
137	Name       string             `json:"name"`
138	Size       int64              `json:"size"`
139	Digest     string             `json:"digest"`
140	ModifiedAt time.Time          `json:"modified_at"`
141	Details    ollamaModelDetails `json:"details"`
142}
143
144type ollamaModelDetails struct {
145	Format     string `json:"format"`
146	Family     string `json:"family"`
147	ParamSize  string `json:"parameter_size"`
148	QuantLevel string `json:"quantization_level"`
149}
150
151type ollamaChatRequest struct {
152	Model         string             `json:"model"`
153	Messages      []ollamaReqMessage `json:"messages"`
154	Tools         []ollamaReqTool    `json:"tools,omitempty"`
155	Temperature   float32            `json:"temperature,omitempty"`
156	Stream        bool               `json:"stream"`
157	StreamOptions *chatStreamOptions `json:"stream_options,omitempty"`
158}
159
160type ollamaReqMessage struct {
161	Role       string              `json:"role"`
162	Content    string              `json:"content"`
163	ToolCallID string              `json:"tool_call_id,omitempty"`
164	ToolCalls  []ollamaReqToolCall `json:"tool_calls,omitempty"`
165}
166
167type ollamaReqTool struct {
168	Type     string            `json:"type"`
169	Function ollamaReqFunction `json:"function"`
170}
171
172type ollamaReqFunction struct {
173	Name        string          `json:"name"`
174	Description string          `json:"description"`
175	Parameters  json.RawMessage `json:"parameters"`
176}
177
178type ollamaReqToolCall struct {
179	ID       string            `json:"id"`
180	Type     string            `json:"type"`
181	Function ollamaReqFuncCall `json:"function"`
182}
183
184type ollamaReqFuncCall struct {
185	Name      string `json:"name"`
186	Arguments string `json:"arguments"`
187}
188
189func toOllamaMessages(messages []Message) []ollamaReqMessage {
190	result := make([]ollamaReqMessage, len(messages))
191	for i, m := range messages {
192		msg := ollamaReqMessage{
193			Role:    m.Role,
194			Content: m.Content,
195		}
196
197		if m.ToolCallID != "" {
198			msg.ToolCallID = m.ToolCallID
199		}
200
201		if len(m.ToolCalls) > 0 {
202			msg.ToolCalls = make([]ollamaReqToolCall, len(m.ToolCalls))
203			for j, tc := range m.ToolCalls {
204				msg.ToolCalls[j] = ollamaReqToolCall{
205					ID:   tc.ID,
206					Type: "function",
207					Function: ollamaReqFuncCall{
208						Name:      tc.Name,
209						Arguments: string(tc.Arguments),
210					},
211				}
212			}
213		}
214
215		result[i] = msg
216	}
217	return result
218}
219
220func toOllamaTools(tools []Tool) []ollamaReqTool {
221	emptyParams := json.RawMessage(`{"type":"object","properties":{}}`)
222	result := make([]ollamaReqTool, len(tools))
223	for i, t := range tools {
224		params := t.Parameters
225		if len(params) == 0 {
226			params = emptyParams
227		}
228
229		result[i] = ollamaReqTool{
230			Type: "function",
231			Function: ollamaReqFunction{
232				Name:        t.Name,
233				Description: t.Description,
234				Parameters:  params,
235			},
236		}
237	}
238	return result
239}
240
241func mergeConsecutiveSystemMessages(messages []ollamaReqMessage) []ollamaReqMessage {
242	if len(messages) == 0 {
243		return messages
244	}
245
246	result := make([]ollamaReqMessage, 0, len(messages))
247	var systemBuffer strings.Builder
248	inSystemRun := false
249
250	for i, msg := range messages {
251		if msg.Role == roleSystem {
252			if inSystemRun {
253				systemBuffer.WriteString("\n\n")
254			} else {
255				inSystemRun = true
256			}
257			systemBuffer.WriteString(msg.Content)
258		} else {
259			if inSystemRun {
260				result = append(result, ollamaReqMessage{
261					Role:    roleSystem,
262					Content: systemBuffer.String(),
263				})
264				systemBuffer.Reset()
265				inSystemRun = false
266			}
267			result = append(result, msg)
268		}
269
270		if i == len(messages)-1 && inSystemRun {
271			result = append(result, ollamaReqMessage{
272				Role:    roleSystem,
273				Content: systemBuffer.String(),
274			})
275		}
276	}
277
278	log.Debug().
279		Int("original_count", len(messages)).
280		Int("merged_count", len(result)).
281		Msg("Merged consecutive system messages")
282
283	return result
284}
285
286type chatCompletionStreamResponse struct {
287	Choices []chatCompletionStreamChoice `json:"choices"`
288	Usage   *chatCompletionUsage         `json:"usage,omitempty"`
289}
290
291type chatCompletionUsage struct {
292	PromptTokens     int `json:"prompt_tokens"`
293	CompletionTokens int `json:"completion_tokens"`
294}
295
296type chatStreamOptions struct {
297	IncludeUsage bool `json:"include_usage"`
298}
299
300type chatCompletionStreamChoice struct {
301	Delta        chatCompletionStreamDelta `json:"delta"`
302	FinishReason *string                   `json:"finish_reason"`
303}
304
305type chatCompletionStreamDelta struct {
306	Role             string                   `json:"role,omitempty"`
307	Content          string                   `json:"content,omitempty"`
308	Reasoning        string                   `json:"reasoning,omitempty"`
309	ReasoningContent string                   `json:"reasoning_content,omitempty"`
310	ToolCalls        []chatCompletionToolCall `json:"tool_calls,omitempty"`
311}
312
313type chatCompletionToolCall struct {
314	Index    int                    `json:"index"`
315	ID       string                 `json:"id"`
316	Type     string                 `json:"type"`
317	Function chatCompletionFunction `json:"function"`
318}
319
320type chatCompletionFunction struct {
321	Name      string `json:"name"`
322	Arguments string `json:"arguments"`
323}
324
325type httpRequestConfig struct {
326	client   *http.Client
327	url      string
328	body     []byte
329	headers  map[string]string
330	provider string
331	model    string
332}
333
334var sseRetryDelays = []time.Duration{5 * time.Second, 10 * time.Second, 15 * time.Second}
335
336func httpDoSSE(ctx context.Context, cfg httpRequestConfig) (io.ReadCloser, error) {
337	maxRetries := len(sseRetryDelays)
338	var lastErr error
339
340	for attempt := 0; attempt <= maxRetries; attempt++ {
341		if err := sseRetryWait(ctx, cfg, attempt); err != nil {
342			return nil, err
343		}
344
345		body, err, retry := sseAttempt(ctx, cfg, attempt)
346		if err != nil {
347			return nil, err
348		}
349		if retry != nil {
350			lastErr = retry
351			continue
352		}
353		return body, nil
354	}
355
356	return nil, fmt.Errorf("SSE request failed after %d retries: %w", maxRetries, lastErr)
357}
358
359func sseRetryWait(ctx context.Context, cfg httpRequestConfig, attempt int) error {
360	if attempt == 0 {
361		log.Info().Str("provider", cfg.provider).Str("model", cfg.model).Msg("SSE stream request started")
362		return nil
363	}
364	delay := sseRetryDelays[attempt-1]
365	log.Warn().Str("provider", cfg.provider).Int("attempt", attempt).Dur("delay", delay).Msg("Retrying SSE connection after transient error")
366	select {
367	case <-time.After(delay):
368		return nil
369	case <-ctx.Done():
370		return ctx.Err()
371	}
372}
373
374func isTransientStatus(code int) bool {
375	return code == 429 || code == 500 || code == 502 || code == 503 || code == 504
376}
377
378func sseAttempt(ctx context.Context, cfg httpRequestConfig, attempt int) (io.ReadCloser, error, error) {
379	httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, cfg.url, bytes.NewReader(cfg.body))
380	if err != nil {
381		return nil, err, nil
382	}
383	httpReq.Header.Set("Content-Type", "application/json")
384	httpReq.Header.Set("Accept", "text/event-stream")
385	for k, v := range cfg.headers {
386		httpReq.Header.Set(k, v)
387	}
388
389	resp, err := cfg.client.Do(httpReq)
390	if err != nil {
391		if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
392			return nil, err, nil
393		}
394		return nil, nil, err
395	}
396
397	if isTransientStatus(resp.StatusCode) {
398		payload, _ := io.ReadAll(resp.Body)
399		resp.Body.Close()
400		retryErr := fmt.Errorf("stream request status %d: %s", resp.StatusCode, strings.TrimSpace(string(payload)))
401		log.Warn().Str("provider", cfg.provider).Int("status", resp.StatusCode).Int("attempt", attempt+1).Msg("SSE retryable error")
402		return nil, nil, retryErr
403	}
404
405	if resp.StatusCode < 200 || resp.StatusCode >= 300 {
406		payload, _ := io.ReadAll(resp.Body)
407		resp.Body.Close()
408		return nil, fmt.Errorf("stream request status %d: %s", resp.StatusCode, strings.TrimSpace(string(payload))), nil
409	}
410
411	return resp.Body, nil, nil
412}
413
414func parseSSEStream(ctx context.Context, reader io.Reader, ch chan<- StreamEvent) {
415	scanner := bufio.NewScanner(reader)
416	scanner.Buffer(make([]byte, 0, 64*1024), 512*1024)
417
418	for scanner.Scan() {
419		line := scanner.Text()
420		if !strings.HasPrefix(line, "data: ") {
421			continue
422		}
423		data := strings.TrimPrefix(line, "data: ")
424		if data == "[DONE]" {
425			trySend(ctx, ch, StreamEvent{Type: EventDone})
426			return
427		}
428
429		var chunk chatCompletionStreamResponse
430		if err := json.Unmarshal([]byte(data), &chunk); err != nil {
431			log.Warn().Err(err).Str("data", data).Msg("Failed to parse SSE chunk")
432			continue
433		}
434		if chunk.Usage != nil {
435			trySend(ctx, ch, StreamEvent{
436				Type:         EventUsage,
437				InputTokens:  chunk.Usage.PromptTokens,
438				OutputTokens: chunk.Usage.CompletionTokens,
439			})
440		}
441		if len(chunk.Choices) == 0 {
442			continue
443		}
444
445		if !emitDelta(ctx, ch, chunk.Choices[0].Delta) {
446			return
447		}
448	}
449
450	if err := scanner.Err(); err != nil {
451		trySend(ctx, ch, StreamEvent{Type: EventError, Err: err})
452		return
453	}
454	trySend(ctx, ch, StreamEvent{Type: EventDone})
455}
456
457func emitDelta(ctx context.Context, ch chan<- StreamEvent, delta chatCompletionStreamDelta) bool {
458	reasoning := delta.Reasoning
459	if reasoning == "" {
460		reasoning = delta.ReasoningContent
461	}
462	if reasoning != "" {
463		if !trySend(ctx, ch, StreamEvent{Type: EventReasoningDelta, Content: reasoning}) {
464			return false
465		}
466	}
467	if delta.Content != "" {
468		if !trySend(ctx, ch, StreamEvent{Type: EventContentDelta, Content: delta.Content}) {
469			return false
470		}
471	}
472	for _, tc := range delta.ToolCalls {
473		if tc.Function.Name != "" {
474			if !trySend(ctx, ch, StreamEvent{
475				Type: EventToolCallBegin, ToolCallIndex: tc.Index,
476				ToolCallID: tc.ID, ToolCallName: tc.Function.Name,
477			}) {
478				return false
479			}
480		}
481		if tc.Function.Arguments != "" {
482			if !trySend(ctx, ch, StreamEvent{
483				Type: EventToolCallDelta, ToolCallIndex: tc.Index,
484				ToolCallArgs: tc.Function.Arguments,
485			}) {
486				return false
487			}
488		}
489	}
490	return true
491}
492
493func trySend(ctx context.Context, ch chan<- StreamEvent, evt StreamEvent) bool {
494	select {
495	case ch <- evt:
496		return true
497	case <-ctx.Done():
498		return false
499	}
500}