@@ -1,15 +1,22 @@
11 package provider
22
33 import (
4+ "bufio"
5+ "bytes"
46 "context"
57 "encoding/json"
8+ "errors"
9+ "fmt"
10+ "io"
611 "net/http"
712 "strings"
13+ "time"
814
915 "github.com/rs/zerolog/log"
1016 )
1117
12-// OllamaProvider implements the Provider interface for Ollama.
18+const roleSystem = "system"
19+
1320 type OllamaProvider struct {
1421 name string
1522 baseURL string
@@ -18,8 +25,6 @@ type OllamaProvider struct {
1825 temperature float64
1926 }
2027
21-// NewOllama creates a new Ollama provider.
22-// Ollama exposes an OpenAI-compatible API at /v1.
2328 func NewOllama(endpoint, model string) *OllamaProvider {
2429 return NewOllamaWithTemp("ollama", endpoint, model, 0.7)
2530 }
@@ -36,16 +41,14 @@ func NewOllamaWithTemp(name string, endpoint, model string, temperature float64)
3641 }
3742 }
3843
39-// Name returns the provider identifier.
4044 func (p *OllamaProvider) Name() string {
4145 return p.name
4246 }
4347
44-// ChatStream sends messages with optional tools and returns a channel of streaming events.
4548 func (p *OllamaProvider) ChatStream(ctx context.Context, messages []Message, tools []Tool) (<-chan StreamEvent, error) {
4649 req := ollamaChatRequest{
4750 Model: p.model,
48- Messages: mergeConsecutiveSystemMessagesOllama(toOllamaMessages(messages)),
51+ Messages: mergeConsecutiveSystemMessages(toOllamaMessages(messages)),
4952 Tools: toOllamaTools(tools),
5053 Temperature: float32(p.temperature),
5154 Stream: true,
@@ -60,7 +63,7 @@ func (p *OllamaProvider) ChatStream(ctx context.Context, messages []Message, too
6063 client: p.httpClient,
6164 url: p.baseURL + "/chat/completions",
6265 body: body,
63- provider: "ollama",
66+ provider: p.name,
6467 model: p.model,
6568 })
6669 if err != nil {
@@ -77,9 +80,12 @@ func (p *OllamaProvider) ChatStream(ctx context.Context, messages []Message, too
7780 return ch, nil
7881 }
7982
80-// OLLAMA-SPECIFIC TYPES
81-// These types are custom to Ollama and should NOT be used by other providers.
82-// They differ from OpenAI standard in structure and field names.
83+func (p *OllamaProvider) Close() error {
84+ if p.httpClient != nil {
85+ p.httpClient.CloseIdleConnections()
86+ }
87+ return nil
88+}
8389
8490 type ollamaChatRequest struct {
8591 Model string `json:"model"`
@@ -119,10 +125,6 @@ type ollamaReqFuncCall struct {
119125 Arguments string `json:"arguments"`
120126 }
121127
122-// toOllamaMessages converts provider messages to Ollama's custom request format.
123-//
124-// OLLAMA-SPECIFIC: Uses custom ollamaReqMessage type instead of OpenAI SDK types.
125-// Ollama has its own message format and tool call structure.
126128 func toOllamaMessages(messages []Message) []ollamaReqMessage {
127129 result := make([]ollamaReqMessage, len(messages))
128130 for i, m := range messages {
@@ -154,9 +156,6 @@ func toOllamaMessages(messages []Message) []ollamaReqMessage {
154156 return result
155157 }
156158
157-// toOllamaTools converts provider-agnostic tools to Ollama tool format.
158-// Parameters is passed through as json.RawMessage to preserve deterministic
159-// serialization order (important for KV-cache hit rate).
160159 func toOllamaTools(tools []Tool) []ollamaReqTool {
161160 emptyParams := json.RawMessage(`{"type":"object","properties":{}}`)
162161 result := make([]ollamaReqTool, len(tools))
@@ -178,15 +177,7 @@ func toOllamaTools(tools []Tool) []ollamaReqTool {
178177 return result
179178 }
180179
181-// mergeConsecutiveSystemMessagesOllama merges consecutive system messages into a single message.
182-//
183-// OLLAMA-SPECIFIC BEHAVIOR:
184-// Unlike OpenAI, Ollama allows system messages anywhere in the conversation.
185-// This function merges consecutive system messages IN PLACE without moving them to the start.
186-// This preserves Ollama's flexible system message handling.
187-//
188-// DO NOT use this function for OpenAI-compatible providers - use mergeSystemMessagesOpenAI instead.
189-func mergeConsecutiveSystemMessagesOllama(messages []ollamaReqMessage) []ollamaReqMessage {
180+func mergeConsecutiveSystemMessages(messages []ollamaReqMessage) []ollamaReqMessage {
190181 if len(messages) == 0 {
191182 return messages
192183 }
@@ -197,7 +188,6 @@ func mergeConsecutiveSystemMessagesOllama(messages []ollamaReqMessage) []ollamaR
197188
198189 for i, msg := range messages {
199190 if msg.Role == roleSystem {
200- // Start or continue system message accumulation
201191 if inSystemRun {
202192 systemBuffer.WriteString("\n\n")
203193 } else {
@@ -205,7 +195,6 @@ func mergeConsecutiveSystemMessagesOllama(messages []ollamaReqMessage) []ollamaR
205195 }
206196 systemBuffer.WriteString(msg.Content)
207197 } else {
208- // Flush accumulated system messages if any
209198 if inSystemRun {
210199 result = append(result, ollamaReqMessage{
211200 Role: roleSystem,
@@ -214,11 +203,9 @@ func mergeConsecutiveSystemMessagesOllama(messages []ollamaReqMessage) []ollamaR
214203 systemBuffer.Reset()
215204 inSystemRun = false
216205 }
217- // Add non-system message
218206 result = append(result, msg)
219207 }
220208
221- // Flush at end of array
222209 if i == len(messages)-1 && inSystemRun {
223210 result = append(result, ollamaReqMessage{
224211 Role: roleSystem,
@@ -230,15 +217,223 @@ func mergeConsecutiveSystemMessagesOllama(messages []ollamaReqMessage) []ollamaR
230217 log.Debug().
231218 Int("original_count", len(messages)).
232219 Int("merged_count", len(result)).
233- Msg("Ollama: Merged consecutive system messages")
220+ Msg("Merged consecutive system messages")
234221
235222 return result
236223 }
237224
238-// Close closes idle HTTP connections
239-func (p *OllamaProvider) Close() error {
240- if p.httpClient != nil {
241- p.httpClient.CloseIdleConnections()
225+type chatCompletionStreamResponse struct {
226+ Choices []chatCompletionStreamChoice `json:"choices"`
227+ Usage *chatCompletionUsage `json:"usage,omitempty"`
228+}
229+
230+type chatCompletionUsage struct {
231+ PromptTokens int `json:"prompt_tokens"`
232+ CompletionTokens int `json:"completion_tokens"`
233+}
234+
235+type chatStreamOptions struct {
236+ IncludeUsage bool `json:"include_usage"`
237+}
238+
239+type chatCompletionStreamChoice struct {
240+ Delta chatCompletionStreamDelta `json:"delta"`
241+ FinishReason *string `json:"finish_reason"`
242+}
243+
244+type chatCompletionStreamDelta struct {
245+ Role string `json:"role,omitempty"`
246+ Content string `json:"content,omitempty"`
247+ Reasoning string `json:"reasoning,omitempty"`
248+ ReasoningContent string `json:"reasoning_content,omitempty"`
249+ ToolCalls []chatCompletionToolCall `json:"tool_calls,omitempty"`
250+}
251+
252+type chatCompletionToolCall struct {
253+ Index int `json:"index"`
254+ ID string `json:"id"`
255+ Type string `json:"type"`
256+ Function chatCompletionFunction `json:"function"`
257+}
258+
259+type chatCompletionFunction struct {
260+ Name string `json:"name"`
261+ Arguments string `json:"arguments"`
262+}
263+
264+type httpRequestConfig struct {
265+ client *http.Client
266+ url string
267+ body []byte
268+ headers map[string]string
269+ provider string
270+ model string
271+}
272+
273+var sseRetryDelays = []time.Duration{5 * time.Second, 10 * time.Second, 15 * time.Second}
274+
275+func httpDoSSE(ctx context.Context, cfg httpRequestConfig) (io.ReadCloser, error) {
276+ maxRetries := len(sseRetryDelays)
277+ var lastErr error
278+
279+ for attempt := 0; attempt <= maxRetries; attempt++ {
280+ if err := sseRetryWait(ctx, cfg, attempt); err != nil {
281+ return nil, err
282+ }
283+
284+ body, err, retry := sseAttempt(ctx, cfg, attempt)
285+ if err != nil {
286+ return nil, err
287+ }
288+ if retry != nil {
289+ lastErr = retry
290+ continue
291+ }
292+ return body, nil
293+ }
294+
295+ return nil, fmt.Errorf("SSE request failed after %d retries: %w", maxRetries, lastErr)
296+}
297+
298+func sseRetryWait(ctx context.Context, cfg httpRequestConfig, attempt int) error {
299+ if attempt == 0 {
300+ log.Info().Str("provider", cfg.provider).Str("model", cfg.model).Msg("SSE stream request started")
301+ return nil
302+ }
303+ delay := sseRetryDelays[attempt-1]
304+ log.Warn().Str("provider", cfg.provider).Int("attempt", attempt).Dur("delay", delay).Msg("Retrying SSE connection after transient error")
305+ select {
306+ case <-time.After(delay):
307+ return nil
308+ case <-ctx.Done():
309+ return ctx.Err()
310+ }
311+}
312+
313+func isTransientStatus(code int) bool {
314+ return code == 429 || code == 500 || code == 502 || code == 503 || code == 504
315+}
316+
317+func sseAttempt(ctx context.Context, cfg httpRequestConfig, attempt int) (io.ReadCloser, error, error) {
318+ httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, cfg.url, bytes.NewReader(cfg.body))
319+ if err != nil {
320+ return nil, err, nil
321+ }
322+ httpReq.Header.Set("Content-Type", "application/json")
323+ httpReq.Header.Set("Accept", "text/event-stream")
324+ for k, v := range cfg.headers {
325+ httpReq.Header.Set(k, v)
326+ }
327+
328+ resp, err := cfg.client.Do(httpReq)
329+ if err != nil {
330+ if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
331+ return nil, err, nil
332+ }
333+ return nil, nil, err
334+ }
335+
336+ if isTransientStatus(resp.StatusCode) {
337+ payload, _ := io.ReadAll(resp.Body)
338+ resp.Body.Close()
339+ retryErr := fmt.Errorf("stream request status %d: %s", resp.StatusCode, strings.TrimSpace(string(payload)))
340+ log.Warn().Str("provider", cfg.provider).Int("status", resp.StatusCode).Int("attempt", attempt+1).Msg("SSE retryable error")
341+ return nil, nil, retryErr
342+ }
343+
344+ if resp.StatusCode < 200 || resp.StatusCode >= 300 {
345+ payload, _ := io.ReadAll(resp.Body)
346+ resp.Body.Close()
347+ return nil, fmt.Errorf("stream request status %d: %s", resp.StatusCode, strings.TrimSpace(string(payload))), nil
348+ }
349+
350+ return resp.Body, nil, nil
351+}
352+
353+func parseSSEStream(ctx context.Context, reader io.Reader, ch chan<- StreamEvent) {
354+ scanner := bufio.NewScanner(reader)
355+ scanner.Buffer(make([]byte, 0, 64*1024), 512*1024)
356+
357+ for scanner.Scan() {
358+ line := scanner.Text()
359+ if !strings.HasPrefix(line, "data: ") {
360+ continue
361+ }
362+ data := strings.TrimPrefix(line, "data: ")
363+ if data == "[DONE]" {
364+ trySend(ctx, ch, StreamEvent{Type: EventDone})
365+ return
366+ }
367+
368+ var chunk chatCompletionStreamResponse
369+ if err := json.Unmarshal([]byte(data), &chunk); err != nil {
370+ log.Warn().Err(err).Str("data", data).Msg("Failed to parse SSE chunk")
371+ continue
372+ }
373+ if chunk.Usage != nil {
374+ trySend(ctx, ch, StreamEvent{
375+ Type: EventUsage,
376+ InputTokens: chunk.Usage.PromptTokens,
377+ OutputTokens: chunk.Usage.CompletionTokens,
378+ })
379+ }
380+ if len(chunk.Choices) == 0 {
381+ continue
382+ }
383+
384+ if !emitDelta(ctx, ch, chunk.Choices[0].Delta) {
385+ return
386+ }
387+ }
388+
389+ if err := scanner.Err(); err != nil {
390+ trySend(ctx, ch, StreamEvent{Type: EventError, Err: err})
391+ return
392+ }
393+ trySend(ctx, ch, StreamEvent{Type: EventDone})
394+}
395+
396+func emitDelta(ctx context.Context, ch chan<- StreamEvent, delta chatCompletionStreamDelta) bool {
397+ reasoning := delta.Reasoning
398+ if reasoning == "" {
399+ reasoning = delta.ReasoningContent
400+ }
401+ if reasoning != "" {
402+ if !trySend(ctx, ch, StreamEvent{Type: EventReasoningDelta, Content: reasoning}) {
403+ return false
404+ }
405+ }
406+ if delta.Content != "" {
407+ if !trySend(ctx, ch, StreamEvent{Type: EventContentDelta, Content: delta.Content}) {
408+ return false
409+ }
410+ }
411+ for _, tc := range delta.ToolCalls {
412+ if tc.Function.Name != "" {
413+ if !trySend(ctx, ch, StreamEvent{
414+ Type: EventToolCallBegin, ToolCallIndex: tc.Index,
415+ ToolCallID: tc.ID, ToolCallName: tc.Function.Name,
416+ }) {
417+ return false
418+ }
419+ }
420+ if tc.Function.Arguments != "" {
421+ if !trySend(ctx, ch, StreamEvent{
422+ Type: EventToolCallDelta, ToolCallIndex: tc.Index,
423+ ToolCallArgs: tc.Function.Arguments,
424+ }) {
425+ return false
426+ }
427+ }
428+ }
429+ return true
430+}
431+
Cut short: this commit's diff is over 1000 lines.