371 lines
11 KiB
Go
371 lines
11 KiB
Go
package llm
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.maximumdirect.net/eric/audita/internal/framework/contracts"
|
|
)
|
|
|
|
const defaultOpenAICompatibleMaxRetries = 3
|
|
|
|
// OpenAICompatibleClientConfig configures the direct HTTP structured-output adapter.
|
|
type OpenAICompatibleClientConfig struct {
|
|
BaseURL string
|
|
Model string
|
|
APIKey string
|
|
MaxRetries int
|
|
HTTPClient *http.Client
|
|
RequestTimeout time.Duration
|
|
}
|
|
|
|
// OpenAICompatibleClient sends OpenAI-compatible chat-completion requests with
|
|
// response_format.type=json_schema.
|
|
type OpenAICompatibleClient struct {
|
|
cfg OpenAICompatibleClientConfig
|
|
httpClient *http.Client
|
|
}
|
|
|
|
var _ contracts.StructuredLLMClient = (*OpenAICompatibleClient)(nil)
|
|
|
|
func NewOpenAICompatibleClient(cfg OpenAICompatibleClientConfig) (*OpenAICompatibleClient, error) {
|
|
normalized, err := normalizeOpenAICompatibleConfig(cfg)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return &OpenAICompatibleClient{
|
|
cfg: normalized,
|
|
httpClient: resolvedHTTPClient(normalized.HTTPClient, normalized.RequestTimeout),
|
|
}, nil
|
|
}
|
|
|
|
func (c *OpenAICompatibleClient) CompleteStructured(
|
|
ctx context.Context,
|
|
req contracts.StructuredCompletionRequest,
|
|
out any,
|
|
) (contracts.StructuredCompletionResponse, error) {
|
|
if err := validateOutputTarget(out); err != nil {
|
|
return contracts.StructuredCompletionResponse{}, err
|
|
}
|
|
|
|
model := strings.TrimSpace(req.Model)
|
|
if model == "" {
|
|
model = c.cfg.Model
|
|
}
|
|
if model == "" {
|
|
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion model must not be empty")
|
|
}
|
|
if req.ResponseSchema == nil {
|
|
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion response schema is required")
|
|
}
|
|
if strings.TrimSpace(req.ResponseSchema.Name) == "" {
|
|
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion response schema name must not be empty")
|
|
}
|
|
if len(req.ResponseSchema.JSONSchema) == 0 || !json.Valid(req.ResponseSchema.JSONSchema) {
|
|
return contracts.StructuredCompletionResponse{}, fmt.Errorf("structured completion response schema JSON must be valid")
|
|
}
|
|
|
|
messages, err := toOpenAICompatibleMessages(req.Messages)
|
|
if err != nil {
|
|
return contracts.StructuredCompletionResponse{}, err
|
|
}
|
|
|
|
endpoint := buildChatCompletionsURL(c.cfg.BaseURL)
|
|
|
|
var lastErr error
|
|
for attempt := 0; attempt <= c.cfg.MaxRetries; attempt++ {
|
|
content, metadata, callErr := c.completeStructuredOnce(
|
|
ctx,
|
|
endpoint,
|
|
model,
|
|
messages,
|
|
req.ResponseSchema.Name,
|
|
req.ResponseSchema.JSONSchema,
|
|
)
|
|
if callErr == nil {
|
|
if decodeErr := json.Unmarshal(content, out); decodeErr != nil {
|
|
callErr = retryableError{err: fmt.Errorf("decode structured output: %w", decodeErr)}
|
|
} else {
|
|
return contracts.StructuredCompletionResponse{
|
|
Content: content,
|
|
Provider: "openai-compatible",
|
|
Model: firstNonEmpty(metadata.Model, model),
|
|
PromptTokens: metadata.PromptTokens,
|
|
CompletionTokens: metadata.CompletionTokens,
|
|
TotalTokens: metadata.TotalTokens,
|
|
}, nil
|
|
}
|
|
}
|
|
|
|
lastErr = sanitizeError(callErr, c.cfg.APIKey)
|
|
if !canRetryFromError(ctx, attempt, c.cfg.MaxRetries, callErr) {
|
|
return contracts.StructuredCompletionResponse{}, lastErr
|
|
}
|
|
}
|
|
|
|
if lastErr == nil {
|
|
lastErr = fmt.Errorf("structured completion failed")
|
|
}
|
|
return contracts.StructuredCompletionResponse{}, lastErr
|
|
}
|
|
|
|
type openAICompatibleMessage struct {
|
|
Role string `json:"role"`
|
|
Content string `json:"content"`
|
|
}
|
|
|
|
type openAICompatibleRequest struct {
|
|
Model string `json:"model"`
|
|
Messages []openAICompatibleMessage `json:"messages"`
|
|
ResponseFormat openAICompatibleStructuredOutputShape `json:"response_format"`
|
|
}
|
|
|
|
type openAICompatibleStructuredOutputShape struct {
|
|
Type string `json:"type"`
|
|
JSONSchema openAICompatibleSchemaEnvelope `json:"json_schema"`
|
|
}
|
|
|
|
type openAICompatibleSchemaEnvelope struct {
|
|
Name string `json:"name"`
|
|
Strict bool `json:"strict"`
|
|
Schema json.RawMessage `json:"schema"`
|
|
}
|
|
|
|
type openAICompatibleChatCompletionsResponse struct {
|
|
Model string `json:"model"`
|
|
Choices []struct {
|
|
Message struct {
|
|
Content json.RawMessage `json:"content"`
|
|
} `json:"message"`
|
|
} `json:"choices"`
|
|
Usage *openAICompatibleUsage `json:"usage,omitempty"`
|
|
}
|
|
|
|
type openAICompatibleUsage struct {
|
|
PromptTokens int `json:"prompt_tokens"`
|
|
CompletionTokens int `json:"completion_tokens"`
|
|
TotalTokens int `json:"total_tokens"`
|
|
}
|
|
|
|
type openAICompatibleResponseMetadata struct {
|
|
Model string
|
|
PromptTokens int
|
|
CompletionTokens int
|
|
TotalTokens int
|
|
}
|
|
|
|
func (c *OpenAICompatibleClient) completeStructuredOnce(
|
|
ctx context.Context,
|
|
endpoint string,
|
|
model string,
|
|
messages []openAICompatibleMessage,
|
|
responseSchemaName string,
|
|
responseSchemaJSON json.RawMessage,
|
|
) (json.RawMessage, openAICompatibleResponseMetadata, error) {
|
|
requestBody := openAICompatibleRequest{
|
|
Model: model,
|
|
Messages: messages,
|
|
ResponseFormat: openAICompatibleStructuredOutputShape{
|
|
Type: "json_schema",
|
|
JSONSchema: openAICompatibleSchemaEnvelope{
|
|
Name: responseSchemaName,
|
|
Strict: true,
|
|
Schema: responseSchemaJSON,
|
|
},
|
|
},
|
|
}
|
|
|
|
payload, err := json.Marshal(requestBody)
|
|
if err != nil {
|
|
return nil, openAICompatibleResponseMetadata{}, fmt.Errorf("marshal provider request: %w", err)
|
|
}
|
|
|
|
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(payload))
|
|
if err != nil {
|
|
return nil, openAICompatibleResponseMetadata{}, fmt.Errorf("build provider request: %w", err)
|
|
}
|
|
httpReq.Header.Set("Content-Type", "application/json")
|
|
if c.cfg.APIKey != "" {
|
|
httpReq.Header.Set("Authorization", "Bearer "+c.cfg.APIKey)
|
|
}
|
|
|
|
httpResp, err := c.httpClient.Do(httpReq)
|
|
if err != nil {
|
|
return nil, openAICompatibleResponseMetadata{}, retryableError{err: fmt.Errorf("provider request failed: %w", err)}
|
|
}
|
|
defer func() {
|
|
_ = httpResp.Body.Close()
|
|
}()
|
|
|
|
rawResp, err := io.ReadAll(httpResp.Body)
|
|
if err != nil {
|
|
return nil, openAICompatibleResponseMetadata{}, retryableError{err: fmt.Errorf("read provider response: %w", err)}
|
|
}
|
|
|
|
if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 {
|
|
statusErr := parseProviderErrorBody(httpResp.StatusCode, rawResp)
|
|
if httpResp.StatusCode == http.StatusTooManyRequests || httpResp.StatusCode >= 500 {
|
|
return nil, openAICompatibleResponseMetadata{}, retryableError{err: statusErr}
|
|
}
|
|
return nil, openAICompatibleResponseMetadata{}, statusErr
|
|
}
|
|
|
|
content, metadata, decodeErr := decodeChatCompletionsResponse(rawResp)
|
|
if decodeErr != nil {
|
|
return nil, openAICompatibleResponseMetadata{}, decodeErr
|
|
}
|
|
return content, metadata, nil
|
|
}
|
|
|
|
func toOpenAICompatibleMessages(messages []contracts.LLMMessage) ([]openAICompatibleMessage, error) {
|
|
result := make([]openAICompatibleMessage, len(messages))
|
|
for i, message := range messages {
|
|
role := strings.TrimSpace(message.Role)
|
|
content := strings.TrimSpace(message.Content)
|
|
if role == "" {
|
|
return nil, fmt.Errorf("message[%d] role must not be empty", i)
|
|
}
|
|
if content == "" {
|
|
return nil, fmt.Errorf("message[%d] content must not be empty", i)
|
|
}
|
|
result[i] = openAICompatibleMessage{
|
|
Role: role,
|
|
Content: content,
|
|
}
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func buildChatCompletionsURL(baseURL string) string {
|
|
return strings.TrimRight(baseURL, "/") + "/chat/completions"
|
|
}
|
|
|
|
func normalizeOpenAICompatibleConfig(cfg OpenAICompatibleClientConfig) (OpenAICompatibleClientConfig, error) {
|
|
cfg.BaseURL = strings.TrimSpace(cfg.BaseURL)
|
|
cfg.Model = strings.TrimSpace(cfg.Model)
|
|
cfg.APIKey = strings.TrimSpace(cfg.APIKey)
|
|
if cfg.MaxRetries == 0 {
|
|
cfg.MaxRetries = defaultOpenAICompatibleMaxRetries
|
|
}
|
|
if cfg.MaxRetries < 0 {
|
|
return OpenAICompatibleClientConfig{}, fmt.Errorf("max retries must be zero or greater")
|
|
}
|
|
if cfg.BaseURL == "" {
|
|
return OpenAICompatibleClientConfig{}, fmt.Errorf("base URL must not be empty")
|
|
}
|
|
if cfg.Model == "" {
|
|
return OpenAICompatibleClientConfig{}, fmt.Errorf("model must not be empty")
|
|
}
|
|
return cfg, nil
|
|
}
|
|
|
|
type retryableError struct {
|
|
err error
|
|
}
|
|
|
|
func (e retryableError) Error() string {
|
|
if e.err == nil {
|
|
return ""
|
|
}
|
|
return e.err.Error()
|
|
}
|
|
|
|
func (e retryableError) Unwrap() error {
|
|
return e.err
|
|
}
|
|
|
|
func canRetryFromError(ctx context.Context, attempt int, maxRetries int, err error) bool {
|
|
if attempt >= maxRetries {
|
|
return false
|
|
}
|
|
if ctx.Err() != nil {
|
|
return false
|
|
}
|
|
var retryable retryableError
|
|
return errors.As(err, &retryable)
|
|
}
|
|
|
|
func firstNonEmpty(values ...string) string {
|
|
for _, value := range values {
|
|
value = strings.TrimSpace(value)
|
|
if value != "" {
|
|
return value
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func decodeChatCompletionsResponse(raw []byte) (json.RawMessage, openAICompatibleResponseMetadata, error) {
|
|
var parsed openAICompatibleChatCompletionsResponse
|
|
if err := json.Unmarshal(raw, &parsed); err != nil {
|
|
return nil, openAICompatibleResponseMetadata{}, retryableError{err: fmt.Errorf("decode provider response envelope: %w", err)}
|
|
}
|
|
if len(parsed.Choices) == 0 {
|
|
return nil, openAICompatibleResponseMetadata{}, fmt.Errorf("provider response missing choices")
|
|
}
|
|
|
|
content, err := extractAssistantContentJSON(parsed.Choices[0].Message.Content)
|
|
if err != nil {
|
|
return nil, openAICompatibleResponseMetadata{}, retryableError{err: err}
|
|
}
|
|
|
|
meta := openAICompatibleResponseMetadata{
|
|
Model: parsed.Model,
|
|
}
|
|
if parsed.Usage != nil {
|
|
meta.PromptTokens = parsed.Usage.PromptTokens
|
|
meta.CompletionTokens = parsed.Usage.CompletionTokens
|
|
meta.TotalTokens = parsed.Usage.TotalTokens
|
|
}
|
|
return content, meta, nil
|
|
}
|
|
|
|
func extractAssistantContentJSON(raw json.RawMessage) (json.RawMessage, error) {
|
|
if len(bytes.TrimSpace(raw)) == 0 || string(bytes.TrimSpace(raw)) == "null" {
|
|
return nil, fmt.Errorf("provider response missing assistant message content")
|
|
}
|
|
|
|
var textContent string
|
|
if err := json.Unmarshal(raw, &textContent); err == nil {
|
|
textContent = strings.TrimSpace(textContent)
|
|
if textContent == "" {
|
|
return nil, fmt.Errorf("provider response assistant message content is empty")
|
|
}
|
|
return json.RawMessage(textContent), nil
|
|
}
|
|
|
|
trimmed := bytes.TrimSpace(raw)
|
|
if json.Valid(trimmed) {
|
|
return append(json.RawMessage(nil), trimmed...), nil
|
|
}
|
|
return nil, fmt.Errorf("provider response assistant message content is not valid JSON")
|
|
}
|
|
|
|
func parseProviderErrorBody(status int, body []byte) error {
|
|
trimmed := strings.TrimSpace(string(body))
|
|
if trimmed == "" {
|
|
return fmt.Errorf("provider returned status %d", status)
|
|
}
|
|
|
|
var payload map[string]any
|
|
if err := json.Unmarshal(body, &payload); err == nil {
|
|
if nested, ok := payload["error"].(map[string]any); ok {
|
|
if msg, ok := nested["message"].(string); ok && strings.TrimSpace(msg) != "" {
|
|
return fmt.Errorf("provider returned status %d: %s", status, strings.TrimSpace(msg))
|
|
}
|
|
}
|
|
if msg, ok := payload["message"].(string); ok && strings.TrimSpace(msg) != "" {
|
|
return fmt.Errorf("provider returned status %d: %s", status, strings.TrimSpace(msg))
|
|
}
|
|
}
|
|
|
|
return fmt.Errorf("provider returned status %d: %s", status, trimmed)
|
|
}
|