62159
Sy
symb / files
symb / internal / provider / ollama.go
12.5 KB
1package provider 2 3import ( 4 "bufio" 5 "bytes" 6 "context" 7 "encoding/json" 8 "errors" 9 "fmt" 10 "io" 11 "net/http" 12 "strings" 13 "time" 14 15 "github.com/rs/zerolog/log" 16) 17 18const roleSystem = "system" 19 20type OllamaProvider struct { 21 name string 22 baseURL string 23 httpClient *http.Client 24 model string 25 temperature float64 26} 27 28func NewOllama(endpoint, model string) *OllamaProvider { 29 return NewOllamaWithTemp("ollama", endpoint, model, 0.7) 30} 31 32func NewOllamaWithTemp(name string, endpoint, model string, temperature float64) *OllamaProvider { 33 baseURL := strings.TrimRight(endpoint, "/") + "/v1" 34 35 return &OllamaProvider{ 36 name: name, 37 baseURL: baseURL, 38 httpClient: &http.Client{}, 39 model: model, 40 temperature: temperature, 41 } 42} 43 44func (p *OllamaProvider) Name() string { 45 return p.name 46} 47 48func (p *OllamaProvider) ChatStream(ctx context.Context, messages []Message, tools []Tool) (<-chan StreamEvent, error) { 49 req := ollamaChatRequest{ 50 Model: p.model, 51 Messages: mergeConsecutiveSystemMessages(toOllamaMessages(messages)), 52 Tools: toOllamaTools(tools), 53 Temperature: float32(p.temperature), 54 Stream: true, 55 StreamOptions: &chatStreamOptions{IncludeUsage: true}, 56 } 57 body, err := json.Marshal(req) 58 if err != nil { 59 return nil, err 60 } 61 62 reader, err := httpDoSSE(ctx, httpRequestConfig{ 63 client: p.httpClient, 64 url: p.baseURL + "/chat/completions", 65 body: body, 66 provider: p.name, 67 model: p.model, 68 }) 69 if err != nil { 70 return nil, err 71 } 72 73 ch := make(chan StreamEvent) 74 go func() { 75 defer close(ch) 76 defer reader.Close() 77 parseSSEStream(ctx, reader, ch) 78 }() 79 80 return ch, nil 81} 82 83func (p *OllamaProvider) ListModels(ctx context.Context) ([]Model, error) { 84 baseURL := strings.TrimRight(p.baseURL, "/v1") 85 url := baseURL + "/api/tags" 86 87 req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) 88 if err != nil { 89 return nil, err 90 } 91 req.Header.Set("Accept", "application/json") 92 93 resp, err := p.httpClient.Do(req) 94 if err != nil { 95 return nil, err 96 } 97 defer resp.Body.Close() 98 99 if resp.StatusCode != http.StatusOK { 100 body, _ := io.ReadAll(resp.Body) 101 return nil, fmt.Errorf("list models status %d: %s", resp.StatusCode, strings.TrimSpace(string(body))) 102 } 103 104 var listResp ollamaListResponse 105 if err := json.NewDecoder(resp.Body).Decode(&listResp); err != nil { 106 return nil, err 107 } 108 109 models := make([]Model, len(listResp.Models)) 110 for i, m := range listResp.Models { 111 models[i] = Model{ 112 Name: m.Name, 113 Size: m.Size, 114 Digest: m.Digest, 115 ModifiedAt: m.ModifiedAt, 116 Format: m.Details.Format, 117 Family: m.Details.Family, 118 ParamSize: m.Details.ParamSize, 119 QuantLevel: m.Details.QuantLevel, 120 } 121 } 122 return models, nil 123} 124 125func (p *OllamaProvider) Close() error { 126 if p.httpClient != nil { 127 p.httpClient.CloseIdleConnections() 128 } 129 return nil 130} 131 132type ollamaListResponse struct { 133 Models []ollamaModel `json:"models"` 134} 135 136type ollamaModel struct { 137 Name string `json:"name"` 138 Size int64 `json:"size"` 139 Digest string `json:"digest"` 140 ModifiedAt time.Time `json:"modified_at"` 141 Details ollamaModelDetails `json:"details"` 142} 143 144type ollamaModelDetails struct { 145 Format string `json:"format"` 146 Family string `json:"family"` 147 ParamSize string `json:"parameter_size"` 148 QuantLevel string `json:"quantization_level"` 149} 150 151type ollamaChatRequest struct { 152 Model string `json:"model"` 153 Messages []ollamaReqMessage `json:"messages"` 154 Tools []ollamaReqTool `json:"tools,omitempty"` 155 Temperature float32 `json:"temperature,omitempty"` 156 Stream bool `json:"stream"` 157 StreamOptions *chatStreamOptions `json:"stream_options,omitempty"` 158} 159 160type ollamaReqMessage struct { 161 Role string `json:"role"` 162 Content string `json:"content"` 163 ToolCallID string `json:"tool_call_id,omitempty"` 164 ToolCalls []ollamaReqToolCall `json:"tool_calls,omitempty"` 165} 166 167type ollamaReqTool struct { 168 Type string `json:"type"` 169 Function ollamaReqFunction `json:"function"` 170} 171 172type ollamaReqFunction struct { 173 Name string `json:"name"` 174 Description string `json:"description"` 175 Parameters json.RawMessage `json:"parameters"` 176} 177 178type ollamaReqToolCall struct { 179 ID string `json:"id"` 180 Type string `json:"type"` 181 Function ollamaReqFuncCall `json:"function"` 182} 183 184type ollamaReqFuncCall struct { 185 Name string `json:"name"` 186 Arguments string `json:"arguments"` 187} 188 189func toOllamaMessages(messages []Message) []ollamaReqMessage { 190 result := make([]ollamaReqMessage, len(messages)) 191 for i, m := range messages { 192 msg := ollamaReqMessage{ 193 Role: m.Role, 194 Content: m.Content, 195 } 196 197 if m.ToolCallID != "" { 198 msg.ToolCallID = m.ToolCallID 199 } 200 201 if len(m.ToolCalls) > 0 { 202 msg.ToolCalls = make([]ollamaReqToolCall, len(m.ToolCalls)) 203 for j, tc := range m.ToolCalls { 204 msg.ToolCalls[j] = ollamaReqToolCall{ 205 ID: tc.ID, 206 Type: "function", 207 Function: ollamaReqFuncCall{ 208 Name: tc.Name, 209 Arguments: string(tc.Arguments), 210 }, 211 } 212 } 213 } 214 215 result[i] = msg 216 } 217 return result 218} 219 220func toOllamaTools(tools []Tool) []ollamaReqTool { 221 emptyParams := json.RawMessage(`{"type":"object","properties":{}}`) 222 result := make([]ollamaReqTool, len(tools)) 223 for i, t := range tools { 224 params := t.Parameters 225 if len(params) == 0 { 226 params = emptyParams 227 } 228 229 result[i] = ollamaReqTool{ 230 Type: "function", 231 Function: ollamaReqFunction{ 232 Name: t.Name, 233 Description: t.Description, 234 Parameters: params, 235 }, 236 } 237 } 238 return result 239} 240 241func mergeConsecutiveSystemMessages(messages []ollamaReqMessage) []ollamaReqMessage { 242 if len(messages) == 0 { 243 return messages 244 } 245 246 result := make([]ollamaReqMessage, 0, len(messages)) 247 var systemBuffer strings.Builder 248 inSystemRun := false 249 250 for i, msg := range messages { 251 if msg.Role == roleSystem { 252 if inSystemRun { 253 systemBuffer.WriteString("\n\n") 254 } else { 255 inSystemRun = true 256 } 257 systemBuffer.WriteString(msg.Content) 258 } else { 259 if inSystemRun { 260 result = append(result, ollamaReqMessage{ 261 Role: roleSystem, 262 Content: systemBuffer.String(), 263 }) 264 systemBuffer.Reset() 265 inSystemRun = false 266 } 267 result = append(result, msg) 268 } 269 270 if i == len(messages)-1 && inSystemRun { 271 result = append(result, ollamaReqMessage{ 272 Role: roleSystem, 273 Content: systemBuffer.String(), 274 }) 275 } 276 } 277 278 log.Debug(). 279 Int("original_count", len(messages)). 280 Int("merged_count", len(result)). 281 Msg("Merged consecutive system messages") 282 283 return result 284} 285 286type chatCompletionStreamResponse struct { 287 Choices []chatCompletionStreamChoice `json:"choices"` 288 Usage *chatCompletionUsage `json:"usage,omitempty"` 289} 290 291type chatCompletionUsage struct { 292 PromptTokens int `json:"prompt_tokens"` 293 CompletionTokens int `json:"completion_tokens"` 294} 295 296type chatStreamOptions struct { 297 IncludeUsage bool `json:"include_usage"` 298} 299 300type chatCompletionStreamChoice struct { 301 Delta chatCompletionStreamDelta `json:"delta"` 302 FinishReason *string `json:"finish_reason"` 303} 304 305type chatCompletionStreamDelta struct { 306 Role string `json:"role,omitempty"` 307 Content string `json:"content,omitempty"` 308 Reasoning string `json:"reasoning,omitempty"` 309 ReasoningContent string `json:"reasoning_content,omitempty"` 310 ToolCalls []chatCompletionToolCall `json:"tool_calls,omitempty"` 311} 312 313type chatCompletionToolCall struct { 314 Index int `json:"index"` 315 ID string `json:"id"` 316 Type string `json:"type"` 317 Function chatCompletionFunction `json:"function"` 318} 319 320type chatCompletionFunction struct { 321 Name string `json:"name"` 322 Arguments string `json:"arguments"` 323} 324 325type httpRequestConfig struct { 326 client *http.Client 327 url string 328 body []byte 329 headers map[string]string 330 provider string 331 model string 332} 333 334var sseRetryDelays = []time.Duration{5 * time.Second, 10 * time.Second, 15 * time.Second} 335 336func httpDoSSE(ctx context.Context, cfg httpRequestConfig) (io.ReadCloser, error) { 337 maxRetries := len(sseRetryDelays) 338 var lastErr error 339 340 for attempt := 0; attempt <= maxRetries; attempt++ { 341 if err := sseRetryWait(ctx, cfg, attempt); err != nil { 342 return nil, err 343 } 344 345 body, err, retry := sseAttempt(ctx, cfg, attempt) 346 if err != nil { 347 return nil, err 348 } 349 if retry != nil { 350 lastErr = retry 351 continue 352 } 353 return body, nil 354 } 355 356 return nil, fmt.Errorf("SSE request failed after %d retries: %w", maxRetries, lastErr) 357} 358 359func sseRetryWait(ctx context.Context, cfg httpRequestConfig, attempt int) error { 360 if attempt == 0 { 361 log.Info().Str("provider", cfg.provider).Str("model", cfg.model).Msg("SSE stream request started") 362 return nil 363 } 364 delay := sseRetryDelays[attempt-1] 365 log.Warn().Str("provider", cfg.provider).Int("attempt", attempt).Dur("delay", delay).Msg("Retrying SSE connection after transient error") 366 select { 367 case <-time.After(delay): 368 return nil 369 case <-ctx.Done(): 370 return ctx.Err() 371 } 372} 373 374func isTransientStatus(code int) bool { 375 return code == 429 || code == 500 || code == 502 || code == 503 || code == 504 376} 377 378func sseAttempt(ctx context.Context, cfg httpRequestConfig, attempt int) (io.ReadCloser, error, error) { 379 httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, cfg.url, bytes.NewReader(cfg.body)) 380 if err != nil { 381 return nil, err, nil 382 } 383 httpReq.Header.Set("Content-Type", "application/json") 384 httpReq.Header.Set("Accept", "text/event-stream") 385 for k, v := range cfg.headers { 386 httpReq.Header.Set(k, v) 387 } 388 389 resp, err := cfg.client.Do(httpReq) 390 if err != nil { 391 if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { 392 return nil, err, nil 393 } 394 return nil, nil, err 395 } 396 397 if isTransientStatus(resp.StatusCode) { 398 payload, _ := io.ReadAll(resp.Body) 399 resp.Body.Close() 400 retryErr := fmt.Errorf("stream request status %d: %s", resp.StatusCode, strings.TrimSpace(string(payload))) 401 log.Warn().Str("provider", cfg.provider).Int("status", resp.StatusCode).Int("attempt", attempt+1).Msg("SSE retryable error") 402 return nil, nil, retryErr 403 } 404 405 if resp.StatusCode < 200 || resp.StatusCode >= 300 { 406 payload, _ := io.ReadAll(resp.Body) 407 resp.Body.Close() 408 return nil, fmt.Errorf("stream request status %d: %s", resp.StatusCode, strings.TrimSpace(string(payload))), nil 409 } 410 411 return resp.Body, nil, nil 412} 413 414func parseSSEStream(ctx context.Context, reader io.Reader, ch chan<- StreamEvent) { 415 scanner := bufio.NewScanner(reader) 416 scanner.Buffer(make([]byte, 0, 64*1024), 512*1024) 417 418 for scanner.Scan() { 419 line := scanner.Text() 420 if !strings.HasPrefix(line, "data: ") { 421 continue 422 } 423 data := strings.TrimPrefix(line, "data: ") 424 if data == "[DONE]" { 425 trySend(ctx, ch, StreamEvent{Type: EventDone}) 426 return 427 } 428 429 var chunk chatCompletionStreamResponse 430 if err := json.Unmarshal([]byte(data), &chunk); err != nil { 431 log.Warn().Err(err).Str("data", data).Msg("Failed to parse SSE chunk") 432 continue 433 } 434 if chunk.Usage != nil { 435 trySend(ctx, ch, StreamEvent{ 436 Type: EventUsage, 437 InputTokens: chunk.Usage.PromptTokens, 438 OutputTokens: chunk.Usage.CompletionTokens, 439 }) 440 } 441 if len(chunk.Choices) == 0 { 442 continue 443 } 444 445 if !emitDelta(ctx, ch, chunk.Choices[0].Delta) { 446 return 447 } 448 } 449 450 if err := scanner.Err(); err != nil { 451 trySend(ctx, ch, StreamEvent{Type: EventError, Err: err}) 452 return 453 } 454 trySend(ctx, ch, StreamEvent{Type: EventDone}) 455} 456 457func emitDelta(ctx context.Context, ch chan<- StreamEvent, delta chatCompletionStreamDelta) bool { 458 reasoning := delta.Reasoning 459 if reasoning == "" { 460 reasoning = delta.ReasoningContent 461 } 462 if reasoning != "" { 463 if !trySend(ctx, ch, StreamEvent{Type: EventReasoningDelta, Content: reasoning}) { 464 return false 465 } 466 } 467 if delta.Content != "" { 468 if !trySend(ctx, ch, StreamEvent{Type: EventContentDelta, Content: delta.Content}) { 469 return false 470 } 471 } 472 for _, tc := range delta.ToolCalls { 473 if tc.Function.Name != "" { 474 if !trySend(ctx, ch, StreamEvent{ 475 Type: EventToolCallBegin, ToolCallIndex: tc.Index, 476 ToolCallID: tc.ID, ToolCallName: tc.Function.Name, 477 }) { 478 return false 479 } 480 } 481 if tc.Function.Arguments != "" { 482 if !trySend(ctx, ch, StreamEvent{ 483 Type: EventToolCallDelta, ToolCallIndex: tc.Index, 484 ToolCallArgs: tc.Function.Arguments, 485 }) { 486 return false 487 } 488 } 489 } 490 return true 491} 492 493func trySend(ctx context.Context, ch chan<- StreamEvent, evt StreamEvent) bool { 494 select { 495 case ch <- evt: 496 return true 497 case <-ctx.Done(): 498 return false 499 } 500}