eric/promptkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Separate frozen HTTP preparation from attempts

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-23 16:43:51 +00:00
parent d31a563ca1
commit 65d5894648
4 changed files with 380 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files

187
internal/llm/openai_compatible_client.go
View File

@@ -7,9 +7,11 @@ import (
"errors"
"fmt"
"io"
"math"
"net/http"
"net/url"
"os"
"strconv"
"strings"
"time"
@@ -87,6 +89,40 @@ func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleCli
}
func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
prepared, err := c.PrepareGeneration(req)
if err != nil {
return nil, err
}
outcome := prepared.Execute(ctx)
if outcome.Err != nil {
return nil, outcome.Err
}
return outcome.Response, nil
}
// PreparedGeneration holds one encoded request and resolved authentication for
// repeated attempts. Each Execute call creates a fresh HTTP request and context.
type PreparedGeneration struct {
endpoint string
payload []byte
apiKey string
timeoutSeconds int
httpClient *http.Client
}
// AttemptOutcome describes one HTTP execution without exposing response bodies
// or headers to the orchestration layer.
type AttemptOutcome struct {
Response *domain.GenerateResponse
Err error
StatusCode int
Usage domain.TokenUsage
UsagePresence *domain.TokenUsagePresence
RetryNotBefore time.Time
}
// PrepareGeneration validates and freezes one built-in request before dispatch.
func (c *OpenAICompatibleClient) PrepareGeneration(req domain.GenerateRequest) (*PreparedGeneration, error) {
if err := domain.ValidateExecutionTargetSettings(req.Target); err != nil {
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
}
@@ -118,22 +154,6 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
if err != nil {
return nil, fmt.Errorf("%w: failed to encode request: %v", ErrRequestFailed, err)
}
requestContext := ctx
if req.Target.TimeoutSeconds > 0 {
var cancel context.CancelFunc
requestContext, cancel = context.WithTimeout(
ctx,
time.Duration(req.Target.TimeoutSeconds)*time.Second,
)
defer cancel()
}
httpReq, err := http.NewRequestWithContext(requestContext, http.MethodPost, endpoint, bytes.NewReader(payload))
if err != nil {
return nil, fmt.Errorf("%w: failed to create request: %v", ErrRequestFailed, err)
}
httpReq.Header.Set("Content-Type", "application/json")
apiKey := strings.TrimSpace(req.Target.APIKey)
envName := strings.TrimSpace(req.Target.APIKeyEnv)
if apiKey == "" && envName != "" {
@@ -145,62 +165,133 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
}
return nil, fmt.Errorf("%w: api key is required", ErrInvalidRequest)
}
if apiKey != "" {
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
}
httpClient := c.httpClient
if httpClient == nil {
httpClient = &http.Client{Timeout: defaults.LLMRequestTimeoutDefault}
}
return &PreparedGeneration{
endpoint: endpoint, payload: payload, apiKey: apiKey,
timeoutSeconds: req.Target.TimeoutSeconds, httpClient: httpClient,
}, nil
}
httpResp, err := httpClient.Do(httpReq)
if err != nil {
return nil, &requestFailedError{cause: err}
// Execute dispatches one attempt. The returned response is non-nil only when
// the successful envelope contains a usable first choice.
func (p *PreparedGeneration) Execute(ctx context.Context) AttemptOutcome {
requestContext := ctx
if p.timeoutSeconds > 0 {
var cancel context.CancelFunc
requestContext, cancel = context.WithTimeout(ctx, time.Duration(p.timeoutSeconds)*time.Second)
defer cancel()
}
httpReq, err := http.NewRequestWithContext(requestContext, http.MethodPost, p.endpoint, bytes.NewReader(p.payload))
if err != nil {
return AttemptOutcome{Err: fmt.Errorf("%w: failed to create request: %v", ErrRequestFailed, err)}
}
httpReq.Header.Set("Content-Type", "application/json")
if p.apiKey != "" {
httpReq.Header.Set("Authorization", "Bearer "+p.apiKey)
}
httpResp, err := p.httpClient.Do(httpReq)
if err != nil {
return AttemptOutcome{Err: &requestFailedError{cause: err}}
}
receivedAt := time.Now()
defer httpResp.Body.Close()
outcome := AttemptOutcome{StatusCode: httpResp.StatusCode}
if httpResp.StatusCode < 200 || httpResp.StatusCode >= 300 {
return nil, providerHTTPErrorFromBody(
outcome.RetryNotBefore = retryNotBefore(httpResp.Header.Values("Retry-After"), receivedAt)
outcome.Err = providerHTTPErrorFromBody(
httpResp.StatusCode,
httpResp.ContentLength,
httpResp.Body,
)
if err := observedContextError(requestContext, ctx, outcome.Err); err != nil {
outcome.Err = err
}
return outcome
}
if httpResp.ContentLength > maxOpenAIChatResponseBytes {
return nil, openAIChatResponseTooLargeError()
outcome.Err = openAIChatResponseTooLargeError()
return outcome
}
wireResp, err := decodeOpenAIChatResponse(httpResp.Body)
if err != nil {
return nil, err
outcome.Err = err
if contextErr := observedContextError(requestContext, ctx, err); contextErr != nil {
outcome.Err = contextErr
}
return outcome
}
outcome.Usage = domain.TokenUsage{
PromptTokens: tokenCount(wireResp.Usage.PromptTokens), CompletionTokens: tokenCount(wireResp.Usage.CompletionTokens),
TotalTokens: tokenCount(wireResp.Usage.TotalTokens), CachedTokens: tokenCount(wireResp.Usage.PromptTokensDetails.CachedTokens),
CacheWriteTokens: tokenCount(wireResp.Usage.CacheWriteTokens),
}
outcome.UsagePresence = &domain.TokenUsagePresence{
PromptTokens: wireResp.Usage.PromptTokens != nil, CompletionTokens: wireResp.Usage.CompletionTokens != nil,
TotalTokens: wireResp.Usage.TotalTokens != nil, CachedTokens: wireResp.Usage.PromptTokensDetails.CachedTokens != nil,
CacheWriteTokens: wireResp.Usage.CacheWriteTokens != nil,
}
if len(wireResp.Choices) == 0 {
return nil, fmt.Errorf("%w: no choices returned", ErrMalformedResponse)
outcome.Err = fmt.Errorf("%w: no choices returned", ErrMalformedResponse)
return outcome
}
content := wireResp.Choices[0].Message.Content
if content == nil {
return nil, fmt.Errorf("%w: first choice has missing message content", ErrMalformedResponse)
outcome.Err = fmt.Errorf("%w: first choice has missing message content", ErrMalformedResponse)
return outcome
}
outcome.Response = &domain.GenerateResponse{Content: *content, Usage: outcome.Usage, UsagePresence: outcome.UsagePresence}
return outcome
}
return &domain.GenerateResponse{
Content: *content,
Usage: domain.TokenUsage{
PromptTokens: tokenCount(wireResp.Usage.PromptTokens),
CompletionTokens: tokenCount(wireResp.Usage.CompletionTokens),
TotalTokens: tokenCount(wireResp.Usage.TotalTokens),
CachedTokens: tokenCount(wireResp.Usage.PromptTokensDetails.CachedTokens),
CacheWriteTokens: tokenCount(wireResp.Usage.CacheWriteTokens),
},
UsagePresence: &domain.TokenUsagePresence{
PromptTokens: wireResp.Usage.PromptTokens != nil,
CompletionTokens: wireResp.Usage.CompletionTokens != nil,
TotalTokens: wireResp.Usage.TotalTokens != nil,
CachedTokens: wireResp.Usage.PromptTokensDetails.CachedTokens != nil,
CacheWriteTokens: wireResp.Usage.CacheWriteTokens != nil,
},
}, nil
func observedContextError(attempt, caller context.Context, cause error) error {
if err := caller.Err(); err != nil {
return &requestFailedError{cause: err}
}
if err := attempt.Err(); err != nil {
return &requestFailedError{cause: err}
}
if errors.Is(cause, context.Canceled) || errors.Is(cause, context.DeadlineExceeded) {
return &requestFailedError{cause: cause}
}
return nil
}
func retryNotBefore(values []string, receivedAt time.Time) time.Time {
if len(values) == 0 {
return time.Time{}
}
value := strings.TrimSpace(values[0])
if value == "" {
return time.Time{}
}
digits := true
for _, r := range value {
if r < '0' || r > '9' {
digits = false
break
}
}
if digits {
seconds, err := strconv.ParseUint(value, 10, 64)
if err != nil || seconds > uint64(math.MaxInt64/int64(time.Second)) {
return receivedAt.Add(time.Duration(math.MaxInt64))
}
return receivedAt.Add(time.Duration(seconds) * time.Second)
}
if when, err := http.ParseTime(value); err == nil {
if when.Before(receivedAt) {
return receivedAt
}
return when
}
return time.Time{}
}
func tokenCount(value *int) int {
@@ -219,6 +310,9 @@ func decodeOpenAIChatResponse(body io.Reader) (openAIChatResponse, error) {
var response openAIChatResponse
if err := decoder.Decode(&response); err != nil {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return openAIChatResponse{}, &requestFailedError{cause: err}
}
if limited.N == 0 {
return openAIChatResponse{}, openAIChatResponseTooLargeError()
}
@@ -230,6 +324,9 @@ func decodeOpenAIChatResponse(body io.Reader) (openAIChatResponse, error) {
var trailing any
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
return openAIChatResponse{}, &requestFailedError{cause: err}
}
if limited.N == 0 {
return openAIChatResponse{}, openAIChatResponseTooLargeError()
}