Implement layered timeout enforcement

This commit is contained in:
2026-07-26 18:31:06 -05:00
parent a0e905ce46
commit ed0c9f6370
12 changed files with 262 additions and 137 deletions

View File

@@ -68,7 +68,9 @@ The optional `model` object accepts `endpoint`, `model`, `temperature`,
`reasoning_effort`, `api_key_env`, and `extra_params`. Numeric ranges and `reasoning_effort`, `api_key_env`, and `extra_params`. Numeric ranges and
credential supply are defined by the [configuration reference](config.md). credential supply are defined by the [configuration reference](config.md).
Explicit zero values for the numeric fields are overrides; zero Explicit zero values for the numeric fields are overrides; zero
`timeout_seconds` disables the outbound client timeout. `timeout_seconds` disables the per-generation deadline only, retaining the
request context and configured transport cap. The timeout layers are defined in
the [outbound integration contract](integrations/openai-compatible-chat.md#authentication-and-timeout).
Raw API-key values are not accepted. `api_key` and any other unknown model Raw API-key values are not accepted. `api_key` and any other unknown model
field cause `400 invalid_json`. field cause `400 invalid_json`.

View File

@@ -64,10 +64,12 @@ Deprecated aliases: `--prompt-id` for `--prompt`, and `--profile-id` for
`--profile`. `--profile`.
Omitted numeric runtime flags preserve the selected effective value; explicit Omitted numeric runtime flags preserve the selected effective value; explicit
zero values override it. `--timeout 0s` disables the outbound HTTP-client zero values override it. `--timeout 0s` disables the per-generation deadline
timeout. CLI durations are converted to whole seconds by truncation toward only; the caller context and configured transport cap remain active. CLI
zero, so any duration whose absolute value is below one second becomes an durations are converted to whole seconds by truncation toward zero, so any
explicit zero-second override. duration whose absolute value is below one second becomes an explicit
zero-second override. The timeout layers are defined in the
[outbound integration contract](integrations/openai-compatible-chat.md#authentication-and-timeout).
There is no raw API-key flag. Use `--api-key-env`. There is no raw API-key flag. Use `--api-key-env`.

View File

@@ -96,7 +96,7 @@ selected by YAML `id`; nested directories are organizational. See the
| `temperature` | no | Number from `0` through `2`. | | `temperature` | no | Number from `0` through `2`. |
| `max_tokens` | no | Integer zero or greater. | | `max_tokens` | no | Integer zero or greater. |
| `top_p` | no | Number from `0` through `1`. | | `top_p` | no | Number from `0` through `1`. |
| `timeout_seconds` | no | Integer zero or greater. | | `timeout_seconds` | no | Per-generation-call deadline in whole seconds; integer zero or greater. |
| `service_tier` | no | Non-empty provider-specific request tier. | | `service_tier` | no | Non-empty provider-specific request tier. |
| `reasoning_effort` | no | Non-empty provider-specific reasoning setting. | | `reasoning_effort` | no | Non-empty provider-specific reasoning setting. |
| `api_key_env` | no | Environment-variable name containing the API key. | | `api_key_env` | no | Environment-variable name containing the API key. |
@@ -105,7 +105,11 @@ selected by YAML `id`; nested directories are organizational. See the
Execution defaults before profile and request overrides are `temperature: 0`, Execution defaults before profile and request overrides are `temperature: 0`,
`max_tokens: 0`, `top_p: 1`, and `timeout_seconds: 600`. Profile numeric values `max_tokens: 0`, `top_p: 1`, and `timeout_seconds: 600`. Profile numeric values
merge by non-zero value. Request overrides preserve presence, so an explicit merge by non-zero value. Request overrides preserve presence, so an explicit
zero can override a profile value. zero can override a profile value. For `timeout_seconds`, explicit request zero
disables the generation deadline while retaining the caller context and the
built-in client's transport cap. See the
[OpenAI-compatible integration contract](integrations/openai-compatible-chat.md#authentication-and-timeout)
for the complete timeout interaction.
Custom profiles take precedence over built-ins with the same ID. Invalid custom Custom profiles take precedence over built-ins with the same ID. Invalid custom
profiles are errors; they do not fall back to a built-in profile. Raw `api_key` profiles are errors; they do not fall back to a built-in profile. Raw `api_key`

View File

@@ -20,16 +20,17 @@ fields:
| `PromptDir` | Prompt-definition directory, required unless a prompt source option is supplied. | | `PromptDir` | Prompt-definition directory, required unless a prompt source option is supplied. |
| `ProfileDir` | Optional custom profile directory over built-ins. | | `ProfileDir` | Optional custom profile directory over built-ins. |
| `SchemaDir` | Schema directory; empty uses `.`. | | `SchemaDir` | Schema directory; empty uses `.`. |
| `Timeout` | Base timeout for the built-in OpenAI-compatible client when `HTTPClient` is absent or has a zero timeout. A non-positive value uses the internal default. | | `Timeout` | Transport-wide safety cap for the built-in OpenAI-compatible client when `HTTPClient` is absent or has a non-positive timeout. A non-positive value uses the internal ten-minute default. |
| `HTTPClient` | Optional HTTP client for that built-in client. It is cloned; a non-zero `Timeout` on it takes precedence over `Config.Timeout` as the base timeout. | | `HTTPClient` | Optional HTTP client for that built-in client. It is cloned; a positive `Timeout` on it is the transport cap and takes precedence over `Config.Timeout`. A non-positive client timeout is treated as unset. |
Nil options are ignored. Invalid construction, including Nil options are ignored. Invalid construction, including
`WithLLMClient(nil)`, returns an error matching `ErrInvalidConfig`. `WithLLMClient(nil)`, returns an error matching `ErrInvalidConfig`.
An effective positive `timeout_seconds` replaces the base timeout. An explicit Profile and request `timeout_seconds` values select a per-generation-call
request override of zero disables the HTTP-client timeout. The timeout is deadline independently of the transport cap. An explicit request override of
otherwise inherited from the supplied client, `Config.Timeout`, or the internal zero disables that generation deadline only. The complete interaction with the
default in that order. caller context is defined in the
[OpenAI-compatible integration contract](../integrations/openai-compatible-chat.md#authentication-and-timeout).
Source options replace their matching directory source: Source options replace their matching directory source:

View File

@@ -77,12 +77,22 @@ resolves the configured non-empty `api_key_env` at request time and sends the
same header. If neither mechanism supplies a key, it sends no same header. If neither mechanism supplies a key, it sends no
`Authorization` header. `Authorization` header.
The client base timeout is chosen at construction: a non-zero timeout on a The transport-wide safety cap is chosen at client construction. A positive
supplied `http.Client` takes precedence over `Config.Timeout`; otherwise a timeout on a supplied `http.Client` takes precedence over a positive
positive `Config.Timeout` is used, then the internal default. A positive `Config.Timeout`; if neither is positive, the internal ten-minute default is
effective `timeout_seconds` replaces that base. An explicit request override used. The supplied client is cloned, and zero or negative timeout values are
of zero disables the HTTP-client timeout; negative values are rejected before a treated as unset.
request is sent.
Separately, a positive effective `timeout_seconds` creates a deadline for each
outbound generation call. Its value follows the execution-setting hierarchy:
an explicit request override, then a non-zero profile value, then the
600-second framework default. An explicit request override of zero disables
only this generation deadline. Negative values are rejected before a request
is sent.
The complete observable rule is that the earliest caller-context deadline,
transport cap, or positive generation deadline terminates the call. Transport
and cancellation failures retain the generation-error classification.
## Response Subset And Failures ## Response Subset And Failures

View File

@@ -10,14 +10,14 @@ format and protocol behavior.
## Construction ## Construction
`NewOpenAICompatibleClient` validates a non-empty configured base URL, records `NewOpenAICompatibleClient` validates a non-empty configured base URL, records
an optional default model, and resolves one base timeout. A supplied client with an optional default model, and resolves one transport cap. A supplied client
a non-zero timeout supplies that base; otherwise a positive configured timeout with a positive timeout supplies that cap; otherwise a positive configured
is used, then the internal default. timeout is used, then the internal default.
When callers supply an `http.Client`, construction clones it rather than When callers supply an `http.Client`, construction clones it rather than
mutating the caller's instance. A supplied client with no timeout receives the mutating the caller's instance. A supplied client with a zero or negative
resolved base timeout in the clone. The client stores the trimmed base URL, timeout receives the resolved transport cap in the clone. The client stores the
default model, and cloned client. trimmed base URL, default model, and cloned client.
## Generate Flow ## Generate Flow
@@ -25,13 +25,13 @@ default model, and cloned client.
1. validate the effective timeout and choose the request endpoint; 1. validate the effective timeout and choose the request endpoint;
2. map the domain request to the internal wire-request representation; 2. map the domain request to the internal wire-request representation;
3. validate and flatten extra parameters, encode JSON, and create the HTTP 3. validate and flatten extra parameters and encode JSON;
request; 4. derive a child context when the effective generation timeout is positive,
4. prefer a direct API key, otherwise resolve the configured key environment then create the HTTP request with that context;
5. prefer a direct API key, otherwise resolve the configured key environment
variable; variable;
5. derive a request HTTP client only when an explicit timeout changes the base 6. execute with the construction-time HTTP client, reject non-success status
client; responses without returning
6. execute the request, reject non-success status responses without returning
provider response bodies; and provider response bodies; and
7. decode the response subset into `domain.GenerateResponse`. 7. decode the response subset into `domain.GenerateResponse`.
@@ -58,8 +58,8 @@ error contracts.
## Package-Local Guarantees ## Package-Local Guarantees
- The default-model fallback happens before wire encoding. - The default-model fallback happens before wire encoding.
- Per-request timeout handling clones a configured HTTP client when needed; it - Per-generation timeout handling derives a request context; it never replaces
does not mutate shared client state. or mutates the configured HTTP client's transport cap.
- Direct API keys take precedence over environment lookup within this client. - Direct API keys take precedence over environment lookup within this client.
- Provider response bodies are discarded for non-success status responses. - Provider response bodies are discarded for non-success status responses.
- The client does not implement retries, tool calls, or a stateful session - The client does not implement retries, tool calls, or a stateful session

View File

@@ -58,7 +58,7 @@ The post-completion review found:
1. Decide and document the precedence among: 1. Decide and document the precedence among:
- `Config.Timeout`; - `Config.Timeout`;
- a non-zero timeout on a supplied `http.Client`; and - a positive timeout on a supplied `http.Client`; and
- an explicit per-request `timeout_seconds` override. - an explicit per-request `timeout_seconds` override.
2. Align `internal/llm` with that decision, removing any constructor state that 2. Align `internal/llm` with that decision, removing any constructor state that
is immediately discarded during generation. is immediately discarded during generation.
@@ -134,8 +134,17 @@ This follow-up is complete when:
## Completion Record ## Completion Record
Completed on 2026-07-26. The timeout contract, single-file source contract, Completed on 2026-07-26. The single-file source contract, strict-decoding
strict-decoding boundary, and maintained HTTP example check were reconciled. boundary, and maintained HTTP example check were reconciled. A subsequent
assembled-engine review corrected the timeout work into two independent
layers: a construction-time transport cap and a per-generation context
deadline, with explicit request zero disabling only the latter. Deterministic
public `Engine.Run` coverage now protects the interaction with caller
deadlines, profile and request values, the framework default, `Config.Timeout`,
and supplied HTTP-client caps.
Final validation confirmed local links and paths, both maintained configuration Final validation confirmed local links and paths, both maintained configuration
files through the real loader, the maintained render and Go package examples, files through the real loader, the maintained render and Go package examples,
`go test ./...`, `go vet ./...`, and `go build ./cmd/scriptorium`. `go test ./...`, `go vet ./...`, and a temporary-output build. The Promptkit
Step 1 documentation gate remains complete after this layered-timeout
correction.

View File

@@ -94,8 +94,9 @@ refresh and policy updates are merged and the repository has an agreed,
accurate baseline. accurate baseline.
**Gate status:** Complete as of 2026-07-26. The completed documentation **Gate status:** Complete as of 2026-07-26. The completed documentation
refresh and follow-up verification record are in the [documentation compliance refresh, follow-up verification, and layered-timeout correction record are in
roadmap](documentation.md). the [documentation compliance roadmap](documentation.md). Step 1 remains
complete after that validation.
### Step 2: Record The Architectural Decision And Detailed Boundary ### Step 2: Record The Architectural Decision And Detailed Boundary

View File

@@ -47,7 +47,11 @@ type Config struct {
PromptDir string PromptDir string
ProfileDir string ProfileDir string
SchemaDir string SchemaDir string
// Timeout is the transport-wide safety cap for the built-in LLM client
// when HTTPClient is absent or has a non-positive timeout.
Timeout time.Duration Timeout time.Duration
// HTTPClient is cloned for the built-in LLM client. Its positive Timeout
// takes precedence over Config.Timeout as the transport-wide safety cap.
HTTPClient *http.Client HTTPClient *http.Client
} }

View File

@@ -5,6 +5,7 @@ import (
"encoding/json" "encoding/json"
"errors" "errors"
"fmt" "fmt"
"io"
"math" "math"
"net/http" "net/http"
"net/http/httptest" "net/http/httptest"
@@ -14,6 +15,7 @@ import (
"strings" "strings"
"testing" "testing"
"testing/fstest" "testing/fstest"
"time"
"gitea.maximumdirect.net/eric/scriptorium" "gitea.maximumdirect.net/eric/scriptorium"
) )
@@ -1154,6 +1156,145 @@ func TestOpenAICompatibleProfileRunsThroughNormalProfilePath(t *testing.T) {
} }
} }
func TestEngineRunLayersTransportAndGenerationTimeouts(t *testing.T) {
intPointer := func(value int) *int {
return &value
}
tests := []struct {
name string
configTimeout time.Duration
suppliedClientTimeout time.Duration
profileTimeoutSeconds int
requestTimeoutSeconds *int
callerTimeout time.Duration
wantRemainingAtRequest time.Duration
}{
{
name: "positive supplied client cap takes precedence over config",
configTimeout: 2 * time.Second,
suppliedClientTimeout: 6 * time.Second,
wantRemainingAtRequest: 6 * time.Second,
},
{
name: "zero supplied client timeout inherits config cap",
configTimeout: 5 * time.Second,
wantRemainingAtRequest: 5 * time.Second,
},
{
name: "profile deadline is shorter than transport cap",
suppliedClientTimeout: 6 * time.Second,
profileTimeoutSeconds: 4,
wantRemainingAtRequest: 4 * time.Second,
},
{
name: "request deadline is shorter than profile and transport limits",
suppliedClientTimeout: 6 * time.Second,
profileTimeoutSeconds: 4,
requestTimeoutSeconds: intPointer(2),
wantRemainingAtRequest: 2 * time.Second,
},
{
name: "explicit zero removes generation deadline but retains transport cap",
suppliedClientTimeout: 5 * time.Second,
profileTimeoutSeconds: 2,
requestTimeoutSeconds: intPointer(0),
wantRemainingAtRequest: 5 * time.Second,
},
{
name: "framework default remains layered with shorter transport cap",
configTimeout: 7 * time.Second,
suppliedClientTimeout: 3 * time.Second,
wantRemainingAtRequest: 3 * time.Second,
},
{
name: "caller deadline remains layered with other limits",
suppliedClientTimeout: 6 * time.Second,
profileTimeoutSeconds: 4,
callerTimeout: 2 * time.Second,
wantRemainingAtRequest: 2 * time.Second,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
var (
sawDeadline bool
remaining time.Duration
)
transport := roundTripFunc(func(req *http.Request) (*http.Response, error) {
deadline, ok := req.Context().Deadline()
sawDeadline = ok
if ok {
remaining = time.Until(deadline)
}
return &http.Response{
StatusCode: http.StatusOK,
Status: "200 OK",
Header: make(http.Header),
Body: io.NopCloser(strings.NewReader(
`{"choices":[{"message":{"content":"ok"}}]}`,
)),
Request: req,
}, nil
})
httpClient := &http.Client{
Timeout: tc.suppliedClientTimeout,
Transport: transport,
}
engine, err := scriptorium.NewEngine(scriptorium.Config{
PromptDir: "./examples/prompts",
SchemaDir: "./examples/schemas",
Timeout: tc.configTimeout,
HTTPClient: httpClient,
}, scriptorium.WithProfiles(scriptorium.Profile{
ID: "layered-timeout",
Endpoint: "http://timeout.test/v1",
Model: "timeout-model",
TimeoutSeconds: tc.profileTimeoutSeconds,
}))
if err != nil {
t.Fatalf("expected engine construction to succeed, got %v", err)
}
ctx := context.Background()
cancel := func() {}
if tc.callerTimeout > 0 {
ctx, cancel = context.WithTimeout(ctx, tc.callerTimeout)
}
defer cancel()
_, err = engine.Run(ctx, scriptorium.RunRequest{
PromptID: "generic.markdown_summary",
ProfileID: "layered-timeout",
Inputs: map[string]scriptorium.ArtifactRef{
"transcript": scriptorium.Inline("Rin opens the gate."),
"glossary": scriptorium.Inline("gate: A guarded passage."),
},
Execution: &scriptorium.ExecutionTargetOverride{
TimeoutSeconds: tc.requestTimeoutSeconds,
},
})
if err != nil {
t.Fatalf("expected run to succeed, got %v", err)
}
if !sawDeadline {
t.Fatal("expected outbound request context to have a deadline")
}
const deadlineTolerance = 750 * time.Millisecond
if remaining < tc.wantRemainingAtRequest-deadlineTolerance ||
remaining > tc.wantRemainingAtRequest+50*time.Millisecond {
t.Fatalf(
"unexpected request deadline: remaining=%v want approximately %v",
remaining,
tc.wantRemainingAtRequest,
)
}
})
}
}
func TestOpenAICompatibleProfileDefersExtraParamsValidation(t *testing.T) { func TestOpenAICompatibleProfileDefersExtraParamsValidation(t *testing.T) {
cyclic := map[string]any{} cyclic := map[string]any{}
cyclic["self"] = cyclic cyclic["self"] = cyclic
@@ -1790,6 +1931,12 @@ type fakeLLMClient struct {
requests []scriptorium.GenerateRequest requests []scriptorium.GenerateRequest
} }
type roundTripFunc func(*http.Request) (*http.Response, error)
func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
return f(req)
}
func (f *fakeLLMClient) Generate(_ context.Context, req scriptorium.GenerateRequest) (*scriptorium.GenerateResponse, error) { func (f *fakeLLMClient) Generate(_ context.Context, req scriptorium.GenerateRequest) (*scriptorium.GenerateResponse, error) {
f.requests = append(f.requests, req) f.requests = append(f.requests, req)
if f.err != nil { if f.err != nil {

View File

@@ -55,7 +55,7 @@ func NewOpenAICompatibleClient(cfg OpenAICompatibleConfig) (*OpenAICompatibleCli
var client *http.Client var client *http.Client
if cfg.HTTPClient != nil { if cfg.HTTPClient != nil {
cloned := *cfg.HTTPClient cloned := *cfg.HTTPClient
if cloned.Timeout == 0 { if cloned.Timeout <= 0 {
cloned.Timeout = timeout cloned.Timeout = timeout
} }
client = &cloned client = &cloned
@@ -99,7 +99,17 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
return nil, fmt.Errorf("%w: failed to encode request: %v", ErrRequestFailed, err) return nil, fmt.Errorf("%w: failed to encode request: %v", ErrRequestFailed, err)
} }
httpReq, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(payload)) requestContext := ctx
if req.Target.TimeoutSeconds > 0 {
var cancel context.CancelFunc
requestContext, cancel = context.WithTimeout(
ctx,
time.Duration(req.Target.TimeoutSeconds)*time.Second,
)
defer cancel()
}
httpReq, err := http.NewRequestWithContext(requestContext, http.MethodPost, endpoint, bytes.NewReader(payload))
if err != nil { if err != nil {
return nil, fmt.Errorf("%w: failed to create request: %v", ErrRequestFailed, err) return nil, fmt.Errorf("%w: failed to create request: %v", ErrRequestFailed, err)
} }
@@ -118,18 +128,6 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
if httpClient == nil { if httpClient == nil {
httpClient = &http.Client{Timeout: defaults.LLMRequestTimeoutDefault} httpClient = &http.Client{Timeout: defaults.LLMRequestTimeoutDefault}
} }
if req.Target.TimeoutSeconds > 0 {
effectiveTimeout := time.Duration(req.Target.TimeoutSeconds) * time.Second
if httpClient.Timeout != effectiveTimeout {
cloned := *httpClient
cloned.Timeout = effectiveTimeout
httpClient = &cloned
}
} else if req.TargetPresence.TimeoutSeconds && httpClient.Timeout != 0 {
cloned := *httpClient
cloned.Timeout = 0
httpClient = &cloned
}
httpResp, err := httpClient.Do(httpReq) httpResp, err := httpClient.Do(httpReq)
if err != nil { if err != nil {

View File

@@ -69,6 +69,36 @@ func TestNewOpenAICompatibleClientDoesNotMutateSuppliedNonzeroTimeoutClient(t *t
} }
} }
func TestNewOpenAICompatibleClientTreatsSuppliedNegativeTimeoutAsUnset(t *testing.T) {
transport := http.DefaultTransport
supplied := &http.Client{
Timeout: -time.Second,
Transport: transport,
}
configuredTimeout := 23 * time.Second
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
Timeout: configuredTimeout,
HTTPClient: supplied,
})
if err != nil {
t.Fatalf("unexpected constructor error: %v", err)
}
if supplied.Timeout != -time.Second {
t.Fatalf("expected supplied client timeout to remain negative, got %v", supplied.Timeout)
}
if client.httpClient == supplied {
t.Fatal("expected constructed client to use a cloned HTTP client")
}
if client.httpClient.Timeout != configuredTimeout {
t.Fatalf("expected cloned client timeout %v, got %v", configuredTimeout, client.httpClient.Timeout)
}
if client.httpClient.Transport != transport {
t.Fatal("expected cloned client to preserve the supplied transport")
}
}
func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) { func TestOpenAICompatibleClientGenerateSuccess(t *testing.T) {
type observedRequest struct { type observedRequest struct {
Authorization string Authorization string
@@ -663,32 +693,6 @@ func TestOpenAICompatibleClientOmitsImplicitZeroNumericFields(t *testing.T) {
} }
} }
func TestOpenAICompatibleClientExplicitZeroTimeoutDisablesClientTimeout(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(20 * time.Millisecond)
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: ts.URL + "/v1",
Timeout: time.Nanosecond,
HTTPClient: &http.Client{Timeout: time.Nanosecond},
})
if err != nil {
t.Fatal(err)
}
_, err = client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{Model: "model", TimeoutSeconds: 0},
TargetPresence: domain.ExecutionTargetPresence{TimeoutSeconds: true},
})
if err != nil {
t.Fatalf("expected explicit zero timeout to disable client timeout, got %v", err)
}
}
func TestOpenAICompatibleClientOmittedTimeoutUsesClientTimeout(t *testing.T) { func TestOpenAICompatibleClientOmittedTimeoutUsesClientTimeout(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(20 * time.Millisecond) time.Sleep(20 * time.Millisecond)
@@ -716,34 +720,6 @@ func TestOpenAICompatibleClientOmittedTimeoutUsesClientTimeout(t *testing.T) {
} }
} }
func TestOpenAICompatibleClientSuppliedHTTPClientTimeoutOverridesConfigTimeout(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(25 * time.Millisecond)
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: ts.URL + "/v1",
Model: "model",
Timeout: 5 * time.Millisecond,
HTTPClient: &http.Client{Timeout: 100 * time.Millisecond},
})
if err != nil {
t.Fatal(err)
}
resp, err := client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
})
if err != nil {
t.Fatalf("expected supplied client timeout to allow the request, got %v", err)
}
if resp.Content != "ok" {
t.Fatalf("unexpected response content: %q", resp.Content)
}
}
func TestOpenAICompatibleClientRejectsInvalidExtraParamsBeforeProviderCall(t *testing.T) { func TestOpenAICompatibleClientRejectsInvalidExtraParamsBeforeProviderCall(t *testing.T) {
tests := []struct { tests := []struct {
name string name string
@@ -1029,35 +1005,6 @@ func TestOpenAICompatibleClientTimeout(t *testing.T) {
} }
} }
func TestOpenAICompatibleClientRequestTimeoutOverride(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
time.Sleep(100 * time.Millisecond)
_, _ = w.Write([]byte(`{"choices":[{"message":{"content":"ok"}}]}`))
}))
defer ts.Close()
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: ts.URL + "/v1",
Model: "m",
Timeout: 5 * time.Millisecond,
HTTPClient: &http.Client{Timeout: 50 * time.Millisecond},
})
if err != nil {
t.Fatal(err)
}
resp, err := client.Generate(context.Background(), domain.GenerateRequest{
Prompt: domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hi"}}},
Target: domain.ExecutionTarget{TimeoutSeconds: 1},
})
if err != nil {
t.Fatalf("expected request-level timeout override to succeed, got %v", err)
}
if resp.Content != "ok" {
t.Fatalf("expected response content ok, got %q", resp.Content)
}
}
func TestOpenAICompatibleClientNegativeTimeoutRejected(t *testing.T) { func TestOpenAICompatibleClientNegativeTimeoutRejected(t *testing.T) {
client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{ client, err := NewOpenAICompatibleClient(OpenAICompatibleConfig{
BaseURL: "http://example.com/v1", BaseURL: "http://example.com/v1",