@@ -0,0 +1,117 @@
1+package provider
2+
3+import (
4+ "context"
5+ "encoding/json"
6+ "net/http"
7+ "strings"
8+
9+ openai "github.com/sashabaranov/go-openai"
10+)
11+
12+type vllmChatRequest struct {
13+ Model string `json:"model"`
14+ Messages []openai.ChatCompletionMessage `json:"messages"`
15+ Tools []openai.Tool `json:"tools,omitempty"`
16+ Temperature float32 `json:"temperature,omitempty"`
17+ TopP float32 `json:"top_p,omitempty"`
18+ RepetitionPenalty float32 `json:"repetition_penalty,omitempty"`
19+ MaxTokens int `json:"max_tokens,omitempty"`
20+ Stream bool `json:"stream"`
21+ StreamOptions *chatStreamOptions `json:"stream_options,omitempty"`
22+}
23+
24+// VLLMProvider implements the Provider interface for vLLM.
25+type VLLMProvider struct {
26+ name string
27+ baseURL string
28+ apiKey string
29+ httpClient *http.Client
30+ model string
31+ temperature float64
32+ topP float64
33+ repeatPenalty float64
34+ maxTokens int
35+}
36+
37+// NewVLLM creates a new vLLM provider.
38+func NewVLLM(endpoint, model, apiKey string) *VLLMProvider {
39+ return NewVLLMWithTemp("vllm", endpoint, model, apiKey, Options{Temperature: 0.7})
40+}
41+
42+func NewVLLMWithTemp(name, endpoint, model, apiKey string, opts Options) *VLLMProvider {
43+ baseURL := strings.TrimRight(endpoint, "/")
44+
45+ return &VLLMProvider{
46+ name: name,
47+ baseURL: baseURL,
48+ apiKey: apiKey,
49+ httpClient: &http.Client{},
50+ model: model,
51+ temperature: opts.Temperature,
52+ topP: opts.TopP,
53+ repeatPenalty: opts.RepeatPenalty,
54+ maxTokens: opts.MaxTokens,
55+ }
56+}
57+
58+// Name returns the provider identifier.
59+func (p *VLLMProvider) Name() string {
60+ return p.name
61+}
62+
63+// ChatStream sends messages with optional tools and returns a channel of streaming events.
64+func (p *VLLMProvider) ChatStream(ctx context.Context, messages []Message, tools []Tool) (<-chan StreamEvent, error) {
65+ req := vllmChatRequest{
66+ Model: p.model,
67+ Messages: mergeSystemMessagesOpenAI(toOpenAIMessages(messages)),
68+ Tools: toOpenAITools(tools),
69+ Temperature: float32(p.temperature),
70+ TopP: float32(p.topP),
71+ RepetitionPenalty: float32(p.repeatPenalty),
72+ MaxTokens: p.maxTokens,
73+ Stream: true,
74+ StreamOptions: &chatStreamOptions{IncludeUsage: true},
75+ }
76+ body, err := json.Marshal(req)
77+ if err != nil {
78+ return nil, err
79+ }
80+
81+ reader, err := httpDoSSE(ctx, httpRequestConfig{
82+ client: p.httpClient,
83+ url: p.baseURL + "/chat/completions",
84+ body: body,
85+ headers: p.authHeaders(),
86+ provider: p.name,
87+ model: p.model,
88+ })
89+ if err != nil {
90+ return nil, err
91+ }
92+
93+ ch := make(chan StreamEvent)
94+ go func() {
95+ defer close(ch)
96+ defer reader.Close()
97+ parseSSEStream(ctx, reader, ch)
98+ }()
99+
100+ return ch, nil
101+}
102+
103+// Close closes idle HTTP connections.
104+func (p *VLLMProvider) Close() error {
105+ if p.httpClient != nil {
106+ p.httpClient.CloseIdleConnections()
107+ }
108+ return nil
109+}
110+
111+func (p *VLLMProvider) authHeaders() map[string]string {
112+ headers := make(map[string]string)
113+ if p.apiKey != "" {
114+ headers["Authorization"] = "Bearer " + p.apiKey
115+ }
116+ return headers
117+}