Added a configurable timeout knob

This commit is contained in:
2026-05-04 22:38:20 -05:00
parent ea05945457
commit c5ce270090
15 changed files with 215 additions and 34 deletions

View File

@@ -49,7 +49,7 @@ func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleCli
timeout := cfg.Timeout
if timeout <= 0 {
timeout = 30 * time.Second
timeout = 10 * time.Minute
}
var client *http.Client
@@ -72,6 +72,10 @@ func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleCli
}
func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
if req.Target.TimeoutSeconds < 0 {
return nil, fmt.Errorf("%w: timeout_seconds must be greater than or equal to 0", ErrInvalidRequest)
}
model := strings.TrimSpace(req.Target.Model)
if model == "" {
model = strings.TrimSpace(c.defaultModel)
@@ -122,7 +126,21 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
httpReq.Header.Set("Authorization", "Bearer "+c.apiKey)
}
httpResp, err := c.httpClient.Do(httpReq)
effectiveTimeout := c.timeout
if req.Target.TimeoutSeconds > 0 {
effectiveTimeout = time.Duration(req.Target.TimeoutSeconds) * time.Second
}
httpClient := c.httpClient
if httpClient == nil {
httpClient = &http.Client{Timeout: effectiveTimeout}
} else if httpClient.Timeout != effectiveTimeout {
cloned := *httpClient
cloned.Timeout = effectiveTimeout
httpClient = &cloned
}
httpResp, err := httpClient.Do(httpReq)
if err != nil {
return nil, fmt.Errorf("%w: %v", ErrRequestFailed, err)
}