Compare commits
3 Commits
c239304c2a
...
2d44305a8a
| Author | SHA1 | Date | |
|---|---|---|---|
| 2d44305a8a | |||
| a11c80291e | |||
| 3239567297 |
@@ -30,9 +30,12 @@ type Backend struct {
|
|||||||
// requires an absolute HTTP or HTTPS URL with a host and without user
|
// requires an absolute HTTP or HTTPS URL with a host and without user
|
||||||
// information, a query string, or a fragment. Paths are allowed.
|
// information, a query string, or a fragment. Paths are allowed.
|
||||||
Endpoint string
|
Endpoint string
|
||||||
// APIKeyEnv optionally names the environment variable containing the API
|
// APIKeyEnv optionally names an environment lookup source for an API key.
|
||||||
// key. NewEngine trims it and requires the portable form
|
// NewEngine trims it and requires the portable form [A-Za-z_][A-Za-z0-9_]*.
|
||||||
// [A-Za-z_][A-Za-z0-9_]*. Store only the name, never a credential value.
|
// A direct RunRequest.APIKey takes precedence. When no usable credential is
|
||||||
|
// available, the built-in client omits Authorization; injected clients own
|
||||||
|
// their own credential-resolution behavior. Store only the name, never a
|
||||||
|
// credential value.
|
||||||
APIKeyEnv string
|
APIKeyEnv string
|
||||||
// ExtraParams contains backend-wide request defaults. Values must be
|
// ExtraParams contains backend-wide request defaults. Values must be
|
||||||
// JSON-compatible, finite, acyclic, and keyed by non-empty strings. Keys
|
// JSON-compatible, finite, acyclic, and keyed by non-empty strings. Keys
|
||||||
|
|||||||
@@ -231,7 +231,7 @@ if err != nil {
|
|||||||
|
|
||||||
target := inspection.EffectiveModelParams
|
target := inspection.EffectiveModelParams
|
||||||
if target.APIKeyEnv != "" {
|
if target.APIKeyEnv != "" {
|
||||||
// Apply application policy for the named environment variable.
|
// This is a configured optional environment lookup source.
|
||||||
} else if inspection.APIKeyRequired {
|
} else if inspection.APIKeyRequired {
|
||||||
// Arrange a direct credential before later execution.
|
// Arrange a direct credential before later execution.
|
||||||
}
|
}
|
||||||
@@ -240,9 +240,11 @@ if target.APIKeyEnv != "" {
|
|||||||
Use this configuration-time boundary when only the profile and its target need
|
Use this configuration-time boundary when only the profile and its target need
|
||||||
checking. Use `Prepare` when the application also needs prompt, input, schema,
|
checking. Use `Prepare` when the application also needs prompt, input, schema,
|
||||||
or rendering work; use prepared execution when that work must remain tied to a
|
or rendering work; use prepared execution when that work must remain tied to a
|
||||||
later execution. Inspection reports credential requirements but leaves the
|
later execution. A reported `APIKeyEnv` is a configured optional source, while
|
||||||
timing of credential enforcement to the application. The method's
|
`APIKeyRequired` is the explicit local requirement. The
|
||||||
[GoDoc](../../engine.go) owns its exact result and error contract.
|
[credential format reference](../formats.md#credentials) and the method's
|
||||||
|
[GoDoc](../../engine.go) own the exact precedence, timing, result, and error
|
||||||
|
contracts.
|
||||||
|
|
||||||
### Set A Per-Run Session And Reasoning
|
### Set A Per-Run Session And Reasoning
|
||||||
|
|
||||||
|
|||||||
@@ -185,7 +185,7 @@ extra_params:
|
|||||||
| `timeout_seconds` | no | Per-generation deadline in whole seconds; integer zero or greater. |
|
| `timeout_seconds` | no | Per-generation deadline in whole seconds; integer zero or greater. |
|
||||||
| `service_tier` | no | Provider-specific request tier. |
|
| `service_tier` | no | Provider-specific request tier. |
|
||||||
| `reasoning_effort` | no | Provider-specific reasoning setting. |
|
| `reasoning_effort` | no | Provider-specific reasoning setting. |
|
||||||
| `api_key_env` | no | Name of an environment variable containing the API key. |
|
| `api_key_env` | no | Optional environment-variable lookup source for an API key. |
|
||||||
| `extra_params` | no | JSON-compatible provider-specific outbound fields. |
|
| `extra_params` | no | JSON-compatible provider-specific outbound fields. |
|
||||||
|
|
||||||
Raw `api_key` is prohibited in profile YAML. Store only an environment
|
Raw `api_key` is prohibited in profile YAML. Store only an environment
|
||||||
@@ -318,16 +318,25 @@ schema produces a failed validation result.
|
|||||||
Credential values belong at the request or environment boundary, never in
|
Credential values belong at the request or environment boundary, never in
|
||||||
prompt, profile, schema, or example files:
|
prompt, profile, schema, or example files:
|
||||||
|
|
||||||
- a file profile names an environment variable with `api_key_env`;
|
- a backend or file profile can name an optional environment lookup source
|
||||||
- an in-memory profile may set `APIKeyRequired`;
|
with `APIKeyEnv` or `api_key_env`;
|
||||||
- a request can provide a direct `APIKey` or override `APIKeyEnv`; and
|
- an in-memory profile may set `APIKeyRequired` as an explicit local
|
||||||
|
requirement;
|
||||||
|
- a request can provide a direct `APIKey` or override the optional `APIKeyEnv`
|
||||||
|
source; and
|
||||||
- a direct request key takes precedence over environment lookup.
|
- a direct request key takes precedence over environment lookup.
|
||||||
|
|
||||||
After a direct request key, the credential-source precedence is request
|
After a direct request key, the credential-source precedence is request
|
||||||
`APIKeyEnv`, profile `api_key_env`, then the backend default. An in-memory
|
`APIKeyEnv`, profile `api_key_env`, then the backend default. An in-memory
|
||||||
profile with `APIKeyRequired` clears an inherited backend environment name and
|
profile with `APIKeyRequired` clears an inherited backend environment name and
|
||||||
requires a direct key unless the request explicitly supplies `APIKeyEnv`.
|
requires a direct key unless the request explicitly supplies `APIKeyEnv`.
|
||||||
Promptkit validates required credential availability during preparation.
|
Named environment sources are optional: when the selected source is absent,
|
||||||
|
empty, or whitespace-only, the built-in client omits the `Authorization`
|
||||||
|
header and handles the provider response normally. `APIKeyRequired` is the
|
||||||
|
only explicit local availability requirement. Promptkit validates required
|
||||||
|
credential availability during preparation and rechecks it when a prepared
|
||||||
|
execution runs. Injected clients receive resolved source metadata but define
|
||||||
|
their own credential-resolution behavior.
|
||||||
Direct keys are excluded from JSON results and redacted by public string
|
Direct keys are excluded from JSON results and redacted by public string
|
||||||
formatters. Environment-variable names may appear in prepared metadata, but
|
formatters. Environment-variable names may appear in prepared metadata, but
|
||||||
their values do not.
|
their values do not.
|
||||||
|
|||||||
@@ -31,11 +31,13 @@ does not serialize it in the provider request.
|
|||||||
|
|
||||||
## Authentication
|
## Authentication
|
||||||
|
|
||||||
A non-empty API key supplied directly on the execution target takes
|
A usable API key supplied directly on the execution target takes precedence.
|
||||||
precedence. Otherwise, when an API-key environment-variable name is supplied,
|
Otherwise, when an API-key environment-variable name is supplied, the client
|
||||||
the client reads that variable and requires a non-empty value. The selected
|
reads and trims that variable. A bearer header is sent only when the resolved
|
||||||
key is sent as `Authorization: Bearer <key>`. No authorization header is sent
|
direct or environment credential is non-empty. When neither source is usable,
|
||||||
when neither mechanism is configured.
|
the client omits `Authorization` and handles the provider response normally.
|
||||||
|
An explicitly required target with no usable source is rejected before
|
||||||
|
transport.
|
||||||
|
|
||||||
The target contains the already resolved environment-variable name: an
|
The target contains the already resolved environment-variable name: an
|
||||||
explicit request override takes precedence over profile metadata, which takes
|
explicit request override takes precedence over profile metadata, which takes
|
||||||
|
|||||||
@@ -55,12 +55,14 @@ the target, rendered messages, and structured-output constraint retained by
|
|||||||
executable preparation. Execution does not reopen or rerender consumer
|
executable preparation. Execution does not reopen or rerender consumer
|
||||||
sources.
|
sources.
|
||||||
|
|
||||||
Before backend admission, the runner rechecks that the frozen credential
|
Before backend admission, the runner rechecks a frozen credential
|
||||||
environment-variable name is available. The handle does not retain the
|
environment-variable name only when the target explicitly requires a
|
||||||
environment value; the model client resolves the value visible when generation
|
credential. The handle does not retain the environment value; the model client
|
||||||
begins. A direct request key remains in private execution state only until the
|
resolves the value visible when generation begins. For optional sources with no
|
||||||
claimed execution finishes or an unclaimed handle is discarded. Exact public
|
usable value, the built-in client omits `Authorization` and continues to the
|
||||||
ownership and redaction semantics belong to the
|
provider. A direct request key remains in private execution state only until
|
||||||
|
the claimed execution finishes or an unclaimed handle is discarded. Exact
|
||||||
|
public ownership and redaction semantics belong to the
|
||||||
[`PreparedExecution` GoDoc](../../prepared_execution.go).
|
[`PreparedExecution` GoDoc](../../prepared_execution.go).
|
||||||
|
|
||||||
## Failure Categories
|
## Failure Categories
|
||||||
@@ -68,12 +70,18 @@ ownership and redaction semantics belong to the
|
|||||||
The package preserves distinct error identities for invalid client
|
The package preserves distinct error identities for invalid client
|
||||||
configuration, invalid generation requests, request execution failures,
|
configuration, invalid generation requests, request execution failures,
|
||||||
non-success provider statuses, and malformed successful responses. Provider
|
non-success provider statuses, and malformed successful responses. Provider
|
||||||
response bodies are not included in non-success errors.
|
response bodies are never exposed in raw form through non-success errors.
|
||||||
|
|
||||||
Invalid nonempty configured endpoints are configuration failures. A missing or
|
Invalid nonempty configured endpoints are configuration failures. A missing or
|
||||||
invalid final selected endpoint is an invalid generation request and is
|
invalid final selected endpoint is an invalid generation request and is
|
||||||
rejected before transport.
|
rejected before transport.
|
||||||
|
|
||||||
|
Authentication resolves a trimmed direct key before a trimmed configured
|
||||||
|
environment value. Optional missing, empty, or whitespace-only sources do not
|
||||||
|
block transport and produce no `Authorization` header. An explicitly required
|
||||||
|
target with no usable source is rejected before transport with the existing
|
||||||
|
invalid-request diagnostics.
|
||||||
|
|
||||||
Successful response bodies have a fixed 16 MiB limit enforced by declared
|
Successful response bodies have a fixed 16 MiB limit enforced by declared
|
||||||
length and by reading at most one byte beyond the boundary. The decoder accepts
|
length and by reading at most one byte beyond the boundary. The decoder accepts
|
||||||
exactly one JSON object plus trailing whitespace and EOF. Size overflow,
|
exactly one JSON object plus trailing whitespace and EOF. Size overflow,
|
||||||
|
|||||||
@@ -1,402 +1,292 @@
|
|||||||
# Structured Generation Errors Implementation Plan
|
# Optional API-Key Environment Implementation Plan
|
||||||
|
|
||||||
## Purpose
|
## Purpose
|
||||||
|
|
||||||
Implement the target state defined in the
|
Change Promptkit's credential handling so an effective `APIKeyEnv` names an
|
||||||
[structured generation errors roadmap](structured-generation-errors.md): turn
|
optional credential source rather than implicitly requiring that environment
|
||||||
every non-2xx response from the built-in OpenAI-compatible client into a
|
variable to be populated. When neither a direct key nor a populated optional
|
||||||
bounded, immutable public error that exposes deliberate provider diagnostics
|
environment variable is available, the built-in OpenAI-compatible client must
|
||||||
through `errors.As` while retaining `ErrLLMGenerate` through `errors.Is`.
|
omit the `Authorization` header and let the upstream provider accept or reject
|
||||||
|
the unauthenticated request.
|
||||||
|
|
||||||
This document owns implementation sequencing. The feature roadmap owns the
|
This document owns implementation sequencing for that change. Follow the
|
||||||
consumer intent, public-policy decisions, safety limits, compatibility
|
architecture, documentation, and testing policies under [`docs/policy/`](../policy/)
|
||||||
boundaries, and non-goals. Follow the architecture, documentation, and testing
|
and the task-specific reading guide in [`docs/development.md`](../development.md)
|
||||||
policies under [`docs/policy/`](../policy/) throughout the work.
|
throughout the work.
|
||||||
|
|
||||||
|
## Target Outcome
|
||||||
|
|
||||||
|
For ordinary and prepared execution:
|
||||||
|
|
||||||
|
1. a nonblank direct `RunRequest.APIKey` remains the highest-precedence
|
||||||
|
credential and produces `Authorization: Bearer <key>`;
|
||||||
|
2. otherwise, a nonblank effective `APIKeyEnv` is read when generation begins;
|
||||||
|
3. a nonblank trimmed environment value produces the bearer header;
|
||||||
|
4. an absent, empty, or whitespace-only optional environment value produces no
|
||||||
|
`Authorization` header and does not prevent preparation or execution; and
|
||||||
|
5. the provider response follows the normal success or error path, including
|
||||||
|
public `GenerationError` conversion for a non-2xx response.
|
||||||
|
|
||||||
|
An explicitly required credential retains local validation. `APIKeyRequired`
|
||||||
|
continues to require either a nonblank direct key or a populated environment
|
||||||
|
variable explicitly selected by the request. Missing required credentials
|
||||||
|
fail before provider transport.
|
||||||
|
|
||||||
## Fixed Decisions
|
## Fixed Decisions
|
||||||
|
|
||||||
- The public type is `GenerationError`. Engine-produced values are pointers;
|
- `APIKeyEnv` is a lookup location, not an assertion that authentication is
|
||||||
fields are unexported; no public constructor or mutation API is added.
|
required. This applies whether the name comes from a built-in backend, a
|
||||||
- Public methods are `StatusCode() int`, `ProviderCode() string`,
|
consumer backend, a file profile, or a request override.
|
||||||
`ProviderType() string`, `ProviderMessage() string`, `Error() string`,
|
- An environment value is usable only after `strings.TrimSpace`; unset, empty,
|
||||||
`GoString() string`, and `Unwrap() error`.
|
and whitespace-only values are equivalent and cause header omission when
|
||||||
- Public `Unwrap` returns `ErrLLMGenerate`. Accessors, formatting, and unwrapping
|
optional.
|
||||||
are safe on a nil receiver and a zero value.
|
- Direct keys retain precedence. When a direct key is nonblank, Promptkit does
|
||||||
- Engine-produced `Error()` text is exactly
|
not need the environment value and must not let a missing environment value
|
||||||
`failed to generate output: provider returned HTTP status N`, where `N` is
|
block the request.
|
||||||
the received status. A nil receiver or zero status returns exactly
|
- `APIKeyRequired` remains the only existing explicit local requirement flag.
|
||||||
`failed to generate output`. `GoString()` returns the same redacted text as
|
Do not add a backend field, YAML field, request field, or new public API.
|
||||||
`Error()` so `%#v` cannot reveal unexported provider fields.
|
- With `APIKeyRequired` true:
|
||||||
- Default formatting contains no provider-controlled code, type, message, or
|
- a nonblank direct key satisfies the requirement;
|
||||||
raw response content. The type has no stable JSON representation.
|
- an explicitly selected nonblank `APIKeyEnv` satisfies it only when that
|
||||||
- The recognized body is one JSON document with a top-level object-valued
|
variable currently contains a nonblank value;
|
||||||
`error`. `message` and `type` accept strings; `code` accepts a string or an
|
- no selected environment name retains the existing required-key failure;
|
||||||
exact `json.Number`; supported fields are independent and unknown fields are
|
and
|
||||||
ignored.
|
- a selected but unset environment retains `ErrAPIKeyEnvMissing` and
|
||||||
- The non-success-body limit is 65,536 bytes. Declared oversize bodies are not
|
`ErrInvalidRequest` at the public boundary.
|
||||||
read; other bodies are read through a 65,537-byte bound. Oversize, malformed,
|
- `ErrAPIKeyEnvMissing` remains exported for compatibility but is narrowed to
|
||||||
unrecognized, or unreadable bodies produce status-only detail.
|
a missing environment credential in an explicitly required flow. Optional
|
||||||
- Normalization converts invalid UTF-8, collapses Unicode whitespace, control,
|
missing environment values do not return it.
|
||||||
and format-character runs to one ASCII space, trims the result, omits blank
|
- Optional credential availability is not frozen during preparation. The
|
||||||
values, and produces one-line strings.
|
built-in client reads the selected environment variable for each generation,
|
||||||
- Codes and types longer than 256 Unicode code points are omitted. Messages
|
including repair generation. Prepared execution therefore uses the value
|
||||||
longer than 4,096 code points retain the first 4,095 code points followed by
|
visible when it runs.
|
||||||
`…`, for a total limit of 4,096.
|
- Required prepared execution continues to recheck credential availability
|
||||||
- Only the concrete built-in transport error is converted into a new public
|
before admission. Optional prepared execution proceeds even if the variable
|
||||||
`GenerationError`. Arbitrary injected-client errors are never inspected or
|
becomes unset after preparation.
|
||||||
enriched.
|
- The built-in transport sets no `Authorization` header at all when no usable
|
||||||
- The use-case and domain layers remain provider-neutral. No retryability,
|
credential exists. It must not send `Bearer ` with an empty value.
|
||||||
retry, logging, presentation, provider-specific envelope, header, or success-
|
- Injected `LLMClient` behavior is not redefined. Promptkit stops rejecting an
|
||||||
response behavior is added.
|
optional missing environment before the injected client is called and
|
||||||
|
continues to pass the effective target through the public adapter without
|
||||||
|
resolving or exposing a secret on the client's behalf.
|
||||||
|
- Upstream authentication policy remains upstream. Promptkit adds no provider-
|
||||||
|
specific authentication rules, retry behavior, status mapping, or special
|
||||||
|
handling beyond the existing structured non-2xx error path.
|
||||||
|
|
||||||
## Execution Rules
|
## Execution Rules
|
||||||
|
|
||||||
- Complete the stages in numerical order. Each stage is scoped for one
|
- Complete the stages in numerical order. Each stage is scoped for one coding-
|
||||||
gpt-5.6-terra implementation prompt and must finish its focused tests before
|
agent prompt and must finish its focused tests before the next stage begins.
|
||||||
the next begins.
|
- Treat all stages as one behavior change. Intermediate stages intentionally
|
||||||
- Treat all stages as one feature delivery. Intermediate stages intentionally
|
leave the use-case and transport policies temporarily different; do not tag,
|
||||||
create internal machinery before exposing it; do not release, tag, or claim
|
release, or claim the change is complete until Stage 3 passes.
|
||||||
the feature is available until Stage 5 is complete.
|
- At the start of each stage, inspect the working tree and preserve all
|
||||||
- At the start of each stage, reread the feature roadmap and the task-specific
|
unrelated changes. In particular, merge rather than overwrite any existing
|
||||||
references in [`docs/development.md`](../development.md). Preserve unrelated
|
edits in files touched by this plan.
|
||||||
working-tree changes.
|
- Use controlled transports, local servers, and `t.Setenv`; never contact a
|
||||||
- Use classical behavior tests at the narrowest owner. Table-drive parser and
|
live provider or depend on the developer machine's credentials.
|
||||||
boundary cases, use controlled transports or local servers, and do not
|
- Update existing tests whose asserted policy is changing instead of retaining
|
||||||
duplicate the internal envelope matrix at the root engine boundary.
|
contradictory tests or adding duplicate coverage under new names.
|
||||||
- Do not contact a live provider, add dependencies, commit, tag, push, or edit
|
- Do not add dependencies, commit, tag, push, or edit release documentation
|
||||||
release documentation unless separately instructed.
|
unless separately instructed.
|
||||||
- Current-state prose documentation changes in Stage 5. Public GoDoc changes
|
- Current-state prose documentation changes only in Stage 3, after the runtime
|
||||||
alongside the public declarations in Stage 4 because GoDoc owns that API.
|
behavior exists. GoDoc changes alongside the public contract in that stage.
|
||||||
|
|
||||||
## Stage 1: Add the Internal Structured Status Error and Envelope Parser
|
## Stage 1: Make Use-Case Credential Validation Requirement-Aware
|
||||||
|
|
||||||
### Objective
|
### Objective
|
||||||
|
|
||||||
Create the transport-owned structured value and pure parsing and normalization
|
Stop preparation and execution orchestration from treating every named
|
||||||
logic without changing `OpenAICompatibleClient.Generate` yet.
|
environment variable as required, while preserving the explicit
|
||||||
|
`APIKeyRequired` and prepared-execution contracts.
|
||||||
|
|
||||||
### Implementation
|
### Implementation
|
||||||
|
|
||||||
1. Add `internal/llm/provider_http_error.go`. Keep all provider HTTP mechanics
|
1. Update `validateAPIKey` in `internal/usecase/runner.go` with this policy:
|
||||||
in `internal/llm`; do not add an HTTP error DTO to `internal/domain` or
|
- return success immediately for a nonblank direct key;
|
||||||
`internal/usecase`.
|
- return success when `apiKeyRequired` is false, regardless of whether
|
||||||
2. Define these private constants:
|
`apiKeyEnv` is blank, populated, or missing;
|
||||||
- `maxProviderErrorResponseBytes int64 = 64 << 10`;
|
- when `apiKeyRequired` is true and the trimmed environment name is blank,
|
||||||
- `maxProviderErrorIdentifierRunes = 256`; and
|
return the existing `ErrAPIKeyRequired`;
|
||||||
- `maxProviderErrorMessageRunes = 4096`.
|
- when `apiKeyRequired` is true and the selected environment variable is
|
||||||
3. Add an exported-within-`internal` `ProviderHTTPError` type with unexported
|
unset, empty, or whitespace-only, return the existing wrapped
|
||||||
`statusCode`, `providerCode`, `providerType`, and `providerMessage` fields.
|
`ErrAPIKeyEnvMissing` diagnostic naming the variable; and
|
||||||
The root facade will need to name this concrete type in Stage 4, but no
|
- otherwise return success.
|
||||||
representation is public outside the module's `internal` boundary.
|
2. Keep the calls from preparation and `RunPrepared` in place. Do not move
|
||||||
4. Give `ProviderHTTPError` nil-safe read-only accessors with the same four
|
environment lookup into backend/profile resolution, admission, the domain
|
||||||
names as the planned public type. Implement:
|
model, or the root facade.
|
||||||
- `Error()` as `llm returned non-success status: status=N` when status is
|
3. Do not clear `APIKeyEnv` from effective or prepared metadata merely because
|
||||||
nonzero and `llm returned non-success status` otherwise;
|
its current value is absent. The name remains the configured lookup source;
|
||||||
- `GoString()` by returning `Error()`; and
|
the secret value remains excluded.
|
||||||
- `Unwrap()` by returning `ErrUnexpectedStatus`.
|
4. Preserve credential-source precedence and `APIKeyRequired` merge behavior.
|
||||||
Never include provider-derived strings in either formatter.
|
A request environment override remains capable of satisfying an in-memory
|
||||||
5. Add a private `providerErrorDetails` value and a private constructor that
|
profile's explicit requirement when its value is populated.
|
||||||
builds `*ProviderHTTPError` from a status plus already normalized details.
|
|
||||||
6. Add `parseProviderErrorEnvelope([]byte) providerErrorDetails` with these
|
|
||||||
rules:
|
|
||||||
- use `json.Decoder` with `UseNumber` and require EOF after trailing JSON
|
|
||||||
whitespace;
|
|
||||||
- require a top-level object and object-valued `error` member;
|
|
||||||
- retain supported fields as `json.RawMessage` so each can be decoded and
|
|
||||||
validated independently;
|
|
||||||
- accept string `message` and `type` values;
|
|
||||||
- accept string or `json.Number` `code`, preserving validated number text
|
|
||||||
without float conversion;
|
|
||||||
- ignore unknown fields and treat invalid supported-field values as absent;
|
|
||||||
and
|
|
||||||
- return empty details for malformed framing, a missing or invalid `error`
|
|
||||||
object, or an object with no usable fields.
|
|
||||||
7. Add private normalization helpers that implement the roadmap's UTF-8,
|
|
||||||
single-line, whitespace/control/format handling and exact rune limits. Use
|
|
||||||
rune-aware operations; do not truncate bytes in the middle of UTF-8. Omit
|
|
||||||
overlong code and type identifiers, and truncate overlong messages to 4,095
|
|
||||||
code points plus `…`.
|
|
||||||
|
|
||||||
### Tests
|
### Tests
|
||||||
|
|
||||||
Add `internal/llm/provider_http_error_test.go` in package `llm` with focused,
|
Update the narrow `internal/usecase` behavior owners rather than adding tests
|
||||||
table-driven tests:
|
for the private helper itself:
|
||||||
|
|
||||||
1. `TestProviderHTTPErrorEnvelopeParsing` covers:
|
1. Replace `TestRunnerRunAPIKeyEnvMissingEnvironmentValueFailsClearly` with a
|
||||||
- all supported string fields;
|
test proving an unset optional profile `APIKeyEnv` reaches the injected
|
||||||
- string, integer, fractional, and exponent-form numeric codes without
|
model client and completes successfully. Explicitly set the named variable
|
||||||
float coercion;
|
to an empty value so the test is independent of the process environment.
|
||||||
- `null` and invalid field types handled independently;
|
2. Preserve the existing populated-environment and direct-key-precedence tests.
|
||||||
- unknown top-level and nested fields;
|
3. Preserve `TestRunnerPrepareAPIKeyRequiredFailsWithoutDirectKey` and the
|
||||||
- missing, null, scalar, and empty `error` values;
|
existing direct-key success coverage.
|
||||||
- malformed, truncated, trailing-garbage, and second-document input; and
|
4. Revise the prepared credential test matrix to prove both distinct cases:
|
||||||
- no raw or unsupported metadata retained.
|
- an optional environment value that disappears after
|
||||||
2. `TestProviderErrorTextNormalizationAndLimits` covers valid multibyte text,
|
`PrepareExecution` does not prevent admission and generation; and
|
||||||
invalid UTF-8 replacement, leading/trailing and repeated whitespace,
|
- an `APIKeyRequired` profile using an explicit request `APIKeyEnv` override
|
||||||
newline/tab/control/format characters, blank normalization, exact identifier
|
still fails with `ErrInvalidRequest` and `ErrAPIKeyEnvMissing` if that
|
||||||
and message boundaries, identifier omission one rune over, and rune-safe
|
value disappears before `RunPrepared`, before admission or generation.
|
||||||
message ellipsis one rune over.
|
5. Keep prepared handles single-use in both outcomes and avoid duplicating
|
||||||
3. `TestProviderHTTPErrorIdentityAndFormatting` covers exact accessors,
|
unrelated preparation, validation, or admission assertions.
|
||||||
`errors.Is(err, ErrUnexpectedStatus)`, nil receivers, zero values, and safe
|
|
||||||
`%v`, `%+v`, and `%#v` formatting with distinctive provider markers absent.
|
|
||||||
|
|
||||||
Do not test body reading, HTTP response ownership, the root public type, or
|
|
||||||
assembled engine behavior in this stage.
|
|
||||||
|
|
||||||
### Verification
|
### Verification
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
go test ./internal/llm -run 'TestProvider'
|
go test ./internal/usecase -run 'APIKey|Credential|Prepared'
|
||||||
go test ./internal/llm
|
go test ./internal/usecase
|
||||||
go test ./...
|
go test ./...
|
||||||
```
|
```
|
||||||
|
|
||||||
Stage 1 is complete when the parser and internal error are fully protected but
|
Stage 1 is complete when optional missing environment values no longer block
|
||||||
the live non-2xx branch remains unchanged.
|
the use-case layer and explicit requirements retain their prior identities and
|
||||||
|
timing.
|
||||||
|
|
||||||
**Status:** Complete.
|
**Status:** Complete.
|
||||||
|
|
||||||
## Stage 2: Add the Bounded Non-Success Body Reader
|
## Stage 2: Omit Authentication in the Built-In Client When Optional Keys Are Missing
|
||||||
|
|
||||||
### Objective
|
### Objective
|
||||||
|
|
||||||
Implement and test bounded response-body extraction independently from HTTP
|
Make the OpenAI-compatible transport implement the optional lookup contract at
|
||||||
client integration, keeping status preservation separate from envelope
|
the boundary that owns the outbound `Authorization` header.
|
||||||
validity.
|
|
||||||
|
|
||||||
### Implementation
|
### Implementation
|
||||||
|
|
||||||
1. In `internal/llm/provider_http_error.go`, add a private helper with the
|
1. Refactor the authentication branch in
|
||||||
equivalent contract of:
|
`internal/llm/openai_compatible_client.go` so it resolves one credential in
|
||||||
|
this order:
|
||||||
```go
|
- trimmed `req.Target.APIKey`; then
|
||||||
func providerHTTPErrorFromBody(
|
- the trimmed value of the environment variable named by the trimmed
|
||||||
statusCode int,
|
`req.Target.APIKeyEnv`.
|
||||||
contentLength int64,
|
2. Set `Authorization` to `Bearer <credential>` only when the resolved value is
|
||||||
body io.Reader,
|
nonblank. If the direct key is blank and the optional environment lookup is
|
||||||
) *ProviderHTTPError
|
absent, empty, or whitespace-only, leave the header unset and continue to
|
||||||
```
|
`http.Client.Do`.
|
||||||
|
3. Retain defensive enforcement for direct internal callers of the transport:
|
||||||
2. Always return a nonnil `ProviderHTTPError` carrying `statusCode`.
|
- if `req.Target.APIKeyRequired` is true and no usable source exists, return
|
||||||
3. When `contentLength` is greater than 65,536, return status-only detail
|
`ErrInvalidRequest` before transport;
|
||||||
without reading `body`.
|
- use the existing missing-environment diagnostic when a nonblank
|
||||||
4. Otherwise read through an `io.LimitedReader` capped at 65,537 bytes. Return
|
environment name was selected; and
|
||||||
status-only detail on a read error or when the extra byte is consumed. Do
|
- use a concise required-key diagnostic when no environment name exists.
|
||||||
not parse a bounded prefix of an incomplete oversized body.
|
Do not add an internal or public error type for this branch.
|
||||||
5. For a complete body at or below the limit, call
|
4. Preserve request construction, endpoint validation, timeout behavior,
|
||||||
`parseProviderErrorEnvelope` and construct the error from its normalized
|
response-body ownership, and direct-key redaction. Never serialize
|
||||||
details.
|
`APIKeyEnv`, `APIKeyRequired`, or a credential into the JSON request body.
|
||||||
6. The helper does not close or drain `body`; the HTTP caller retains response-
|
5. Allow an unauthenticated upstream non-2xx response to flow through the
|
||||||
body ownership. It must never read beyond the one-byte overflow probe.
|
existing `ProviderHTTPError` and public `GenerationError` machinery without
|
||||||
|
adding a credential-specific translation.
|
||||||
|
|
||||||
### Tests
|
### Tests
|
||||||
|
|
||||||
Extend `internal/llm/provider_http_error_test.go` with
|
Update `TestOpenAICompatibleClientAuthentication` in
|
||||||
`TestProviderHTTPErrorBodyBounds`, using counting, failing, and guarded readers
|
`internal/llm/openai_compatible_client_test.go` as the transport contract owner.
|
||||||
instead of an HTTP server. Cover:
|
Its table should cover only the meaningful credential states:
|
||||||
|
|
||||||
- an ordinary recognized envelope;
|
- a direct key takes precedence over a populated environment value;
|
||||||
- an exact 65,536-byte body, using trailing JSON whitespace to reach the
|
- a populated environment value supplies the bearer header when no direct key
|
||||||
boundary while remaining one valid document;
|
is present;
|
||||||
- a declared 65,537-byte body with zero reads;
|
- no configured source sends the request without `Authorization`;
|
||||||
- unknown-length and underreported 65,537-byte bodies with exactly 65,537 bytes
|
- a named but unset optional environment sends the request without
|
||||||
read and status-only detail;
|
`Authorization`;
|
||||||
- an early read failure with status-only detail; and
|
- a named but unset required environment fails with `ErrInvalidRequest` before
|
||||||
- an empty body with status-only detail.
|
transport; and
|
||||||
|
- a required target with no source also fails before transport.
|
||||||
|
|
||||||
Assert the provider markers are absent whenever extraction is discarded. Do
|
Use exact header assertions because presence or absence of `Authorization` is
|
||||||
not add body-closure assertions here because this helper does not own closing.
|
the wire contract. Do not repeat endpoint, body, timeout, or response-error
|
||||||
|
matrices in this test.
|
||||||
|
|
||||||
### Verification
|
### Verification
|
||||||
|
|
||||||
```sh
|
```sh
|
||||||
go test ./internal/llm -run 'TestProviderHTTPErrorBodyBounds|TestProvider'
|
go test ./internal/llm -run TestOpenAICompatibleClientAuthentication
|
||||||
go test ./internal/llm
|
go test ./internal/llm
|
||||||
go test ./...
|
go test ./...
|
||||||
```
|
```
|
||||||
|
|
||||||
Stage 2 is complete when every read path is deterministically bounded and the
|
Stage 2 is complete when the built-in transport sends unauthenticated requests
|
||||||
HTTP client's current branch is still untouched.
|
for optional missing sources, still enforces explicit requirements, and all
|
||||||
|
existing provider-response behavior remains green.
|
||||||
|
|
||||||
**Status:** Complete.
|
**Status:** Complete.
|
||||||
|
|
||||||
## Stage 3: Integrate Structured Status Errors into the Built-In Client
|
## Stage 3: Align the Public Contract, Durable Documentation, and Full Validation
|
||||||
|
|
||||||
### Objective
|
### Objective
|
||||||
|
|
||||||
Replace the built-in client's status-only discard branch with the bounded
|
Prove the assembled consumer workflow, update canonical contract owners, and
|
||||||
internal error while preserving all successful, cancellation, transport, and
|
complete repository-wide validation.
|
||||||
body-ownership behavior.
|
|
||||||
|
|
||||||
### Implementation
|
### Public Contract And Tests
|
||||||
|
|
||||||
1. In `internal/llm/openai_compatible_client.go`, replace the non-2xx branch's
|
1. Update the GoDoc for `ErrAPIKeyEnvMissing` in `engine.go` so it applies only
|
||||||
4,096-byte discard and formatted sentinel with
|
when an explicitly required credential names an unset or empty environment
|
||||||
`providerHTTPErrorFromBody(httpResp.StatusCode, httpResp.ContentLength,
|
variable. Retain the exported value and its `ErrInvalidRequest` identity.
|
||||||
httpResp.Body)`.
|
2. Update credential GoDoc where the exact public semantics are exposed:
|
||||||
2. Keep the existing `defer httpResp.Body.Close()` as the single body-closure
|
- `Backend.APIKeyEnv` in `backends.go`;
|
||||||
owner. Do not close in the helper, drain after the bound, or reuse the 16 MiB
|
- `ExecutionTarget.APIKeyEnv` and
|
||||||
successful-response decoder or limit.
|
`ExecutionTargetOverride.APIKeyEnv` in `types.go`; and
|
||||||
3. Return no partial `GenerateResponse` for every non-2xx response.
|
- any nearby `APIKeyRequired` wording that would otherwise imply every
|
||||||
4. Preserve `errors.Is(err, ErrUnexpectedStatus)` through
|
named environment source is mandatory.
|
||||||
`ProviderHTTPError.Unwrap`. Do not change `requestFailedError`, endpoint or
|
State that optional missing values cause the built-in client to omit
|
||||||
request validation, authentication, timeout handling, successful response
|
`Authorization`; do not promise that injected clients resolve environment
|
||||||
decoding, or response-size behavior.
|
variables identically.
|
||||||
|
3. Replace `TestMissingCredentialsFailClearlyWhenProfileRequiresAuth` in
|
||||||
### Tests
|
`engine_test.go`, whose old assertion is intentionally obsolete, with one
|
||||||
|
focused external-package contract test. Assemble a real engine with a
|
||||||
1. Update the existing non-success case in
|
controlled HTTP client or local server, select a backend or file profile
|
||||||
`internal/llm/openai_compatible_client_test.go` to assert
|
whose `APIKeyEnv` is explicitly empty in the process environment, and have
|
||||||
`errors.As(err, &providerHTTPError)`, exact status, and continued
|
the controlled upstream return a structured authentication failure. Assert:
|
||||||
`ErrUnexpectedStatus` identity. Keep the existing raw-body redaction check.
|
- the request reaches upstream;
|
||||||
2. Add `internal/llm/provider_http_error_transport_test.go` with:
|
- `Authorization` is absent;
|
||||||
- `TestOpenAICompatibleClientStructuredNonSuccessResponse`, proving a
|
- the result is nil;
|
||||||
recognized envelope supplies all normalized internal fields and returns
|
- the error does not match `ErrInvalidRequest` or
|
||||||
no generation result;
|
`ErrAPIKeyEnvMissing`; and
|
||||||
- `TestOpenAICompatibleClientNonSuccessBodyOwnership`, table-driving normal,
|
- the error matches `ErrLLMGenerate` and is discoverable as a
|
||||||
declared-oversize, streamed-oversize, underreported, malformed, and read-
|
`*GenerationError` with the upstream status and representative provider
|
||||||
failure cases through a controlled transport; and
|
detail.
|
||||||
- assertions that every body is closed, no case reads beyond its bound,
|
4. Keep this root test representative. The transport authentication table owns
|
||||||
declared oversize performs no read, and discarded details remain empty.
|
the full credential matrix, and the existing generation-error tests own
|
||||||
3. Reuse existing controlled-transport and counting-reader helpers when they
|
envelope parsing, bounds, and formatting.
|
||||||
are clear and package-local. Do not duplicate successful-response framing,
|
|
||||||
timeout, authentication, or endpoint matrices.
|
|
||||||
|
|
||||||
### Verification
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./internal/llm -run 'TestOpenAICompatibleClient.*NonSuccess|TestProvider'
|
|
||||||
go test ./internal/llm
|
|
||||||
go test ./...
|
|
||||||
```
|
|
||||||
|
|
||||||
Stage 3 is complete when the built-in client emits the structured internal
|
|
||||||
error for every non-2xx response and all prior internal identities remain
|
|
||||||
green.
|
|
||||||
|
|
||||||
**Status:** Complete.
|
|
||||||
|
|
||||||
## Stage 4: Add the Public Error and Root Mapping Contract
|
|
||||||
|
|
||||||
### Objective
|
|
||||||
|
|
||||||
Translate only the built-in transport's structured error at the root facade,
|
|
||||||
publish the immutable consumer API, and prove ordinary, prepared, and injected-
|
|
||||||
client behavior.
|
|
||||||
|
|
||||||
### Implementation
|
|
||||||
|
|
||||||
1. Add `generation_error.go` in package `promptkit` with an immutable public
|
|
||||||
`GenerationError` whose four fields are unexported strings or integers. Add
|
|
||||||
one private constructor that accepts the four normalized scalar values. The
|
|
||||||
public error file must not import `internal/llm` or retain the internal error
|
|
||||||
or raw body; `errors.go` performs that adaptation at the facade boundary.
|
|
||||||
2. Implement the four public accessors exactly as fixed above. Each returns
|
|
||||||
zero or empty on a nil receiver.
|
|
||||||
3. Implement `Error()` with the fixed strings from this plan, `GoString()` by
|
|
||||||
returning `Error()`, and `Unwrap()` by returning `ErrLLMGenerate` even for a
|
|
||||||
nil receiver. Do not implement mutable fields, an exported constructor,
|
|
||||||
retry helpers, HTTP mapping, `fmt.Formatter`, or JSON methods.
|
|
||||||
4. Write complete GoDoc covering:
|
|
||||||
- built-in-client and non-2xx scope;
|
|
||||||
- ordinary and prepared execution;
|
|
||||||
- `errors.Is` and pointer-target `errors.As` usage;
|
|
||||||
- immutable and caller-owned semantics;
|
|
||||||
- nil and zero behavior;
|
|
||||||
- lack of stable JSON;
|
|
||||||
- safe default formatting; and
|
|
||||||
- the fact that every provider accessor is untrusted and may contain
|
|
||||||
sensitive request or schema fragments.
|
|
||||||
5. In `errors.go`, after the special capacity conversion and before generic
|
|
||||||
sentinel wrapping, use `errors.As` for a nonnil concrete
|
|
||||||
`*llm.ProviderHTTPError`. Convert it to `*GenerationError` and return that
|
|
||||||
public value directly. Do not parse error text or recognize an interface
|
|
||||||
that an injected client could accidentally satisfy.
|
|
||||||
6. Preserve the existing generic `publicErrorFor` path for all other failures.
|
|
||||||
In particular, arbitrary injected-client errors remain wrapped with
|
|
||||||
`ErrLLMGenerate` and retain their original identity.
|
|
||||||
7. Update public GoDoc in the same stage:
|
|
||||||
- the `ErrLLMGenerate` declaration points to `GenerationError` for built-in
|
|
||||||
non-2xx responses;
|
|
||||||
- `Engine.Run` and `Engine.RunPrepared` mention the typed error without
|
|
||||||
restating its accessors;
|
|
||||||
- `doc.go` distinguishes mutable `CapacityError` from immutable
|
|
||||||
`GenerationError`, lists both as lacking stable JSON, and calls out the
|
|
||||||
provider-detail trust boundary; and
|
|
||||||
- injected `LLMClient` GoDoc remains clear that arbitrary client errors are
|
|
||||||
preserved rather than translated.
|
|
||||||
|
|
||||||
### Tests
|
|
||||||
|
|
||||||
1. Add package-internal tests for `GenerationError` nil and zero receivers,
|
|
||||||
exact fixed formatting, and unwrapping. Do not expose a test-only public
|
|
||||||
constructor or turn the lack of stable JSON into a serialized-output
|
|
||||||
contract.
|
|
||||||
2. Add a focused external-package contract test, preferably in
|
|
||||||
`generation_error_contract_test.go`, that obtains errors through a real
|
|
||||||
assembled engine with a controlled HTTP transport:
|
|
||||||
- an ordinary `Run` with a recognized envelope asserts a nil result,
|
|
||||||
`errors.Is(err, ErrLLMGenerate)`, pointer-target `errors.As`, all four
|
|
||||||
accessors, exact status-bearing formatting, and absence of distinctive
|
|
||||||
code/type/message markers from `%v`, `%+v`, and `%#v`;
|
|
||||||
- one `RunPrepared` case proves the same public type and status cross the
|
|
||||||
prepared boundary without repeating every parser field; and
|
|
||||||
- neither case contacts a live provider or uses a real credential.
|
|
||||||
3. Extend the existing injected-client preservation owner with one assertion
|
|
||||||
that an arbitrary injected error does not become a `*GenerationError`, while
|
|
||||||
still matching both `ErrLLMGenerate` and the injected error.
|
|
||||||
4. Keep detailed envelope, normalization, size, and body-ownership matrices in
|
|
||||||
`internal/llm`; root tests remain representative.
|
|
||||||
|
|
||||||
### Verification
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test . -run 'TestGenerationError|TestBuiltInGenerationError|TestRunAddsLLMGenerate'
|
|
||||||
go test ./internal/llm
|
|
||||||
go test ./...
|
|
||||||
```
|
|
||||||
|
|
||||||
Stage 4 is complete when consumers can inspect built-in non-2xx details through
|
|
||||||
the stable root contract and injected errors remain untouched.
|
|
||||||
|
|
||||||
**Status:** Complete.
|
|
||||||
|
|
||||||
## Stage 5: Update Canonical Documentation and Run Full Validation
|
|
||||||
|
|
||||||
### Objective
|
|
||||||
|
|
||||||
Make durable documentation match the implemented contract, remove the roadmap
|
|
||||||
links that describe parsing as future work, and complete repository-wide
|
|
||||||
validation.
|
|
||||||
|
|
||||||
### Documentation
|
### Documentation
|
||||||
|
|
||||||
1. Update `docs/integrations/openai-compatible-chat.md` as the canonical wire
|
Update each canonical owner only for its topic:
|
||||||
owner. Replace the status-only paragraph with:
|
|
||||||
- the recognized top-level envelope and independent field types;
|
1. In `docs/formats.md`, define profile, backend, and request `api_key_env`
|
||||||
- strict single-document framing and unknown-field behavior;
|
values as optional lookup sources. In the credentials section, distinguish
|
||||||
- the exact 65,536-byte read policy and status-only fallbacks;
|
them from `APIKeyRequired`, document header omission for a missing optional
|
||||||
- exact normalization and field limits;
|
value, retain direct-key precedence, and state that required availability is
|
||||||
- response closure and no-overread behavior; and
|
validated during preparation and rechecked for prepared execution.
|
||||||
- the prohibition on raw bodies, headers, endpoints, credentials, request
|
2. In `docs/integrations/openai-compatible-chat.md`, document the outbound
|
||||||
data, schemas, and generated content.
|
authentication wire behavior: a bearer header is sent only for a usable
|
||||||
2. Update `docs/internal/llm.md` with the internal `ProviderHTTPError`, bounded
|
direct or environment credential; otherwise the header is omitted and the
|
||||||
reader and parser flow, retained `ErrUnexpectedStatus` identity, root
|
provider response is handled normally.
|
||||||
conversion boundary, and narrow test ownership. Remove its link that defers
|
3. In `docs/internal/llm.md`, update the internal flow and failure categories to
|
||||||
parsing to the feature roadmap.
|
distinguish optional omission from explicit required-key rejection. Preserve
|
||||||
3. Update `docs/consumers/pkg-promptkit.md` under `Handle Errors` with one short
|
any unrelated edits already present in this file.
|
||||||
`errors.As` example. Show status and deliberate message access, warn that all
|
4. In `docs/consumers/pkg-promptkit.md`, clarify near profile inspection or
|
||||||
provider fields are untrusted and potentially sensitive, leave retry and
|
credential guidance that a reported `APIKeyEnv` is a configured optional
|
||||||
presentation policy to the application, and link to `GenerationError`
|
source, while `APIKeyRequired` is the explicit local requirement. Link to
|
||||||
GoDoc rather than duplicating its full contract.
|
the exact public GoDoc or format reference rather than reproducing the full
|
||||||
4. Update `docs/internal/overview.md` only enough to inventory implemented
|
precedence contract.
|
||||||
responsibilities: the root facade owns typed capacity and generation error
|
5. Do not change architecture policy, backend IDs/default names, release notes,
|
||||||
mapping, and `internal/llm` owns bounded structured non-success response
|
the README, or examples unless implementation uncovers a concrete
|
||||||
decoding. Do not duplicate limits or accessor details there.
|
inaccurate current-state statement in one of those owners.
|
||||||
5. Do not change `docs/policy/architecture.md`, `docs/formats.md`, backend
|
|
||||||
documentation, release notes, or the README unless implementation reveals a
|
|
||||||
concrete inaccurate statement. Their canonical topics do not own this
|
|
||||||
contract.
|
|
||||||
|
|
||||||
### Final Validation
|
### Final Validation
|
||||||
|
|
||||||
@@ -412,26 +302,31 @@ go run ./examples/go-library/prepare
|
|||||||
go run ./examples/go-library/run
|
go run ./examples/go-library/run
|
||||||
```
|
```
|
||||||
|
|
||||||
Also perform the documented Go-formatting, local Markdown-link, repository-
|
Also run the documented tracked-Go formatting check, local Markdown-link
|
||||||
hygiene, ignored-file, and credential scans. Review both example outputs and
|
validation, repository-hygiene checks, ignored-file check, credential scan, and
|
||||||
confirm that all commands remain deterministic, offline, and credential-free.
|
`git diff --check`. Review the complete diff and confirm that:
|
||||||
|
|
||||||
Finally review the complete diff against the feature roadmap and confirm:
|
- optional absent, empty, and whitespace-only environment values omit
|
||||||
|
authentication and reach transport;
|
||||||
|
- direct and populated environment credentials still produce the correct
|
||||||
|
bearer header;
|
||||||
|
- explicit required flows still fail locally with the intended identities;
|
||||||
|
- ordinary, prepared, built-in, injected-client, and repair paths follow their
|
||||||
|
stated ownership boundaries;
|
||||||
|
- upstream non-2xx responses remain structured generation errors rather than
|
||||||
|
local credential errors;
|
||||||
|
- no credential value is serialized, retained in public metadata, or exposed
|
||||||
|
by formatting; and
|
||||||
|
- durable documentation describes only the now-implemented behavior with one
|
||||||
|
canonical owner per exact contract.
|
||||||
|
|
||||||
- every built-in non-2xx response produces a status-bearing public type;
|
Stage 3 is complete when the public workflow, documentation, and full
|
||||||
- malformed and oversized bodies cannot erase status or leak partial content;
|
maintainer validation all match the target outcome.
|
||||||
- provider-derived strings appear only through deliberate accessors;
|
|
||||||
- default and Go-syntax formatting are redacted;
|
|
||||||
- successful, cancellation, capacity, validation, repair, and injected-client
|
|
||||||
behavior is unchanged;
|
|
||||||
- no provider policy entered the use-case or domain layers; and
|
|
||||||
- each exact contract has one canonical documentation and test owner.
|
|
||||||
|
|
||||||
**Status:** Complete.
|
**Status:** Complete.
|
||||||
|
|
||||||
## Open Questions
|
## Open Questions
|
||||||
|
|
||||||
None. The roadmap and this plan fix the public API, internal representation,
|
None. Optional environment lookup, explicit requirement behavior, precedence,
|
||||||
wire envelope, normalization and bounds, status fallback, formatting,
|
prepared-execution timing, outbound header semantics, error compatibility,
|
||||||
propagation, compatibility, documentation, and verification decisions required
|
documentation ownership, and test boundaries are fixed by this plan.
|
||||||
for implementation.
|
|
||||||
|
|||||||
@@ -53,9 +53,9 @@ var (
|
|||||||
// an execution profile or resolve its backend, except for the profile
|
// an execution profile or resolve its backend, except for the profile
|
||||||
// not-found case represented by ErrProfileNotFound.
|
// not-found case represented by ErrProfileNotFound.
|
||||||
ErrProfileLoad = errors.New("failed to load execution profile")
|
ErrProfileLoad = errors.New("failed to load execution profile")
|
||||||
// ErrAPIKeyEnvMissing identifies an APIKeyEnv whose environment variable is
|
// ErrAPIKeyEnvMissing identifies an explicitly required APIKeyEnv whose
|
||||||
// unset or empty when no direct RunRequest.APIKey takes precedence. Such an
|
// environment variable is unset or empty after direct RunRequest.APIKey
|
||||||
// error also matches ErrInvalidRequest.
|
// precedence is applied. Such an error also matches ErrInvalidRequest.
|
||||||
ErrAPIKeyEnvMissing = errors.New("api_key_env points to an unset environment variable")
|
ErrAPIKeyEnvMissing = errors.New("api_key_env points to an unset environment variable")
|
||||||
// ErrArtifactLoad identifies a failure to resolve an input artifact. Errors
|
// ErrArtifactLoad identifies a failure to resolve an input artifact. Errors
|
||||||
// returned by an injected ArtifactReader remain available through errors.Is.
|
// returned by an injected ArtifactReader remain available through errors.Is.
|
||||||
|
|||||||
@@ -977,37 +977,72 @@ func TestPrepareDirectAPIKeyBypassesMissingEnvWithoutLeakingOrHashing(t *testing
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMissingCredentialsFailClearlyWhenProfileRequiresAuth(t *testing.T) {
|
func TestOptionalMissingCredentialsReachUpstream(t *testing.T) {
|
||||||
const missingEnv = "PROMPTKIT_PUBLIC_AUTH_MISSING"
|
const missingEnv = "PROMPTKIT_PUBLIC_AUTH_MISSING"
|
||||||
|
const providerBody = `{"error":{"message":"authentication failed","type":"authentication_error","code":"invalid_api_key"}}`
|
||||||
t.Setenv(missingEnv, "")
|
t.Setenv(missingEnv, "")
|
||||||
|
|
||||||
profileDir := t.TempDir()
|
called := false
|
||||||
writePublicProfileFileWithAPIKeyEnv(t, profileDir, "requires-auth", "http://localhost:8000/v1", "test-model", missingEnv)
|
config := promptkit.Config{
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
PromptDir: frameworkPromptDir,
|
||||||
PromptDir: frameworkPromptDir,
|
SchemaDir: frameworkSchemaDir,
|
||||||
ProfileDir: profileDir,
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
SchemaDir: frameworkSchemaDir,
|
called = true
|
||||||
})
|
if values := req.Header.Values("Authorization"); len(values) != 0 {
|
||||||
|
t.Fatalf("Authorization values = %q, want absent", values)
|
||||||
|
}
|
||||||
|
return &http.Response{
|
||||||
|
StatusCode: http.StatusUnauthorized,
|
||||||
|
ContentLength: int64(len(providerBody)),
|
||||||
|
Body: io.NopCloser(strings.NewReader(providerBody)),
|
||||||
|
}, nil
|
||||||
|
})},
|
||||||
|
}
|
||||||
|
engine, err := promptkit.NewEngine(config,
|
||||||
|
promptkit.WithBackend(promptkit.Backend{
|
||||||
|
ID: "optional-auth",
|
||||||
|
Endpoint: "http://provider.test/v1",
|
||||||
|
APIKeyEnv: missingEnv,
|
||||||
|
}),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "optional-auth-profile",
|
||||||
|
BackendID: "optional-auth",
|
||||||
|
Model: "test-model",
|
||||||
|
}),
|
||||||
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected engine construction to succeed, got %v", err)
|
t.Fatalf("expected engine construction to succeed, got %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = engine.Prepare(context.Background(), promptkit.RunRequest{
|
result, err := engine.Run(context.Background(), promptkit.RunRequest{
|
||||||
PromptID: frameworkMarkdownSummaryPromptID,
|
PromptID: frameworkMarkdownSummaryPromptID,
|
||||||
ProfileID: "requires-auth",
|
ProfileID: "optional-auth-profile",
|
||||||
Inputs: map[string]promptkit.ArtifactRef{
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
"transcript": promptkit.Inline("Rin opens the gate."),
|
"transcript": promptkit.Inline("Rin opens the gate."),
|
||||||
"glossary": promptkit.Inline("gate: A guarded passage."),
|
"glossary": promptkit.Inline("gate: A guarded passage."),
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if !errors.Is(err, promptkit.ErrInvalidRequest) {
|
if !called {
|
||||||
t.Fatalf("expected invalid request for missing credentials, got %v", err)
|
t.Fatal("optional missing credential did not reach upstream")
|
||||||
}
|
}
|
||||||
if !errors.Is(err, promptkit.ErrAPIKeyEnvMissing) {
|
if result != nil {
|
||||||
t.Fatalf("expected missing credential environment error, got %v", err)
|
t.Fatalf("result = %+v, want nil", result)
|
||||||
}
|
}
|
||||||
if err == nil || !strings.Contains(err.Error(), missingEnv) {
|
if errors.Is(err, promptkit.ErrInvalidRequest) || errors.Is(err, promptkit.ErrAPIKeyEnvMissing) {
|
||||||
t.Fatalf("expected missing env name in error, got %v", err)
|
t.Fatalf("error = %v, want upstream generation error without credential identities", err)
|
||||||
|
}
|
||||||
|
if !errors.Is(err, promptkit.ErrLLMGenerate) {
|
||||||
|
t.Fatalf("error = %v, want ErrLLMGenerate", err)
|
||||||
|
}
|
||||||
|
var generationErr *promptkit.GenerationError
|
||||||
|
if !errors.As(err, &generationErr) {
|
||||||
|
t.Fatalf("error = %v, want GenerationError", err)
|
||||||
|
}
|
||||||
|
if generationErr.StatusCode() != http.StatusUnauthorized ||
|
||||||
|
generationErr.ProviderType() != "authentication_error" ||
|
||||||
|
generationErr.ProviderCode() != "invalid_api_key" ||
|
||||||
|
generationErr.ProviderMessage() != "authentication failed" {
|
||||||
|
t.Fatalf("GenerationError = %+v, want structured upstream authentication failure", generationErr)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -21,16 +21,20 @@ func mapPublicError(err error) error {
|
|||||||
strings.TrimSpace(internalCapacityError.BackendID) != "" {
|
strings.TrimSpace(internalCapacityError.BackendID) != "" {
|
||||||
return &CapacityError{BackendID: internalCapacityError.BackendID}
|
return &CapacityError{BackendID: internalCapacityError.BackendID}
|
||||||
}
|
}
|
||||||
|
publicErr := publicErrorFor(err)
|
||||||
var providerHTTPError *llm.ProviderHTTPError
|
var providerHTTPError *llm.ProviderHTTPError
|
||||||
if errors.As(err, &providerHTTPError) && providerHTTPError != nil {
|
if errors.As(err, &providerHTTPError) && providerHTTPError != nil {
|
||||||
return newGenerationError(
|
generationErr := newGenerationError(
|
||||||
providerHTTPError.StatusCode(),
|
providerHTTPError.StatusCode(),
|
||||||
providerHTTPError.ProviderCode(),
|
providerHTTPError.ProviderCode(),
|
||||||
providerHTTPError.ProviderType(),
|
providerHTTPError.ProviderType(),
|
||||||
providerHTTPError.ProviderMessage(),
|
providerHTTPError.ProviderMessage(),
|
||||||
)
|
)
|
||||||
|
if publicErr != nil && !errors.Is(publicErr, ErrLLMGenerate) {
|
||||||
|
return fmt.Errorf("%w: %w", publicErr, generationErr)
|
||||||
|
}
|
||||||
|
return generationErr
|
||||||
}
|
}
|
||||||
publicErr := publicErrorFor(err)
|
|
||||||
if publicErr == nil {
|
if publicErr == nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -48,3 +49,27 @@ func TestMapPublicErrorTranslatesCapacityError(t *testing.T) {
|
|||||||
t.Fatalf("mapped backend ID changed with source error: %q", publicErr.BackendID)
|
t.Fatalf("mapped backend ID changed with source error: %q", publicErr.BackendID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMapPublicErrorPreservesValidationAroundGenerationError(t *testing.T) {
|
||||||
|
internalErr := fmt.Errorf(
|
||||||
|
"%w: %w",
|
||||||
|
usecase.ErrValidation,
|
||||||
|
&llm.ProviderHTTPError{},
|
||||||
|
)
|
||||||
|
|
||||||
|
err := mapPublicError(internalErr)
|
||||||
|
if !errors.Is(err, ErrValidation) {
|
||||||
|
t.Fatalf("mapped error=%v, want ErrValidation", err)
|
||||||
|
}
|
||||||
|
if !errors.Is(err, ErrLLMGenerate) {
|
||||||
|
t.Fatalf("mapped error=%v, want ErrLLMGenerate", err)
|
||||||
|
}
|
||||||
|
var generationErr *GenerationError
|
||||||
|
if !errors.As(err, &generationErr) || generationErr == nil {
|
||||||
|
t.Fatalf("mapped error=%v, want GenerationError", err)
|
||||||
|
}
|
||||||
|
var leakedInternalErr *llm.ProviderHTTPError
|
||||||
|
if errors.As(err, &leakedInternalErr) {
|
||||||
|
t.Fatalf("mapped error exposes internal ProviderHTTPError: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -133,13 +133,18 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
|||||||
return nil, fmt.Errorf("%w: failed to create request: %v", ErrRequestFailed, err)
|
return nil, fmt.Errorf("%w: failed to create request: %v", ErrRequestFailed, err)
|
||||||
}
|
}
|
||||||
httpReq.Header.Set("Content-Type", "application/json")
|
httpReq.Header.Set("Content-Type", "application/json")
|
||||||
if apiKey := strings.TrimSpace(req.Target.APIKey); apiKey != "" {
|
apiKey := strings.TrimSpace(req.Target.APIKey)
|
||||||
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
|
envName := strings.TrimSpace(req.Target.APIKeyEnv)
|
||||||
} else if envName := strings.TrimSpace(req.Target.APIKeyEnv); envName != "" {
|
if apiKey == "" && envName != "" {
|
||||||
apiKey := strings.TrimSpace(os.Getenv(envName))
|
apiKey = strings.TrimSpace(os.Getenv(envName))
|
||||||
if apiKey == "" {
|
}
|
||||||
|
if apiKey == "" && req.Target.APIKeyRequired {
|
||||||
|
if envName != "" {
|
||||||
return nil, fmt.Errorf("%w: api key environment variable %q is not set", ErrInvalidRequest, envName)
|
return nil, fmt.Errorf("%w: api key environment variable %q is not set", ErrInvalidRequest, envName)
|
||||||
}
|
}
|
||||||
|
return nil, fmt.Errorf("%w: api key is required", ErrInvalidRequest)
|
||||||
|
}
|
||||||
|
if apiKey != "" {
|
||||||
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
|
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -501,36 +501,61 @@ func checkCompleteRequestAndResponseMapping(t *testing.T) {
|
|||||||
|
|
||||||
func TestOpenAICompatibleClientAuthentication(t *testing.T) {
|
func TestOpenAICompatibleClientAuthentication(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
configureEnv func(*testing.T)
|
configureEnv func(*testing.T)
|
||||||
target domain.ExecutionTarget
|
target domain.ExecutionTarget
|
||||||
wantAuth string
|
wantAuthorization string
|
||||||
wantErr error
|
wantError error
|
||||||
wantCallCount int
|
wantCallCount int
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "direct key takes precedence over environment",
|
name: "direct key takes precedence over environment",
|
||||||
configureEnv: func(t *testing.T) {
|
configureEnv: func(t *testing.T) {
|
||||||
t.Setenv("PROMPTKIT_TEST_API_KEY", "env-key")
|
t.Setenv("PROMPTKIT_TEST_API_KEY", " env-key ")
|
||||||
},
|
},
|
||||||
target: domain.ExecutionTarget{
|
target: domain.ExecutionTarget{
|
||||||
APIKeyEnv: "PROMPTKIT_TEST_API_KEY",
|
APIKeyEnv: "PROMPTKIT_TEST_API_KEY",
|
||||||
APIKey: "direct-llm-key",
|
APIKey: " direct-llm-key ",
|
||||||
},
|
},
|
||||||
wantAuth: "Bearer direct-llm-key",
|
wantAuthorization: "Bearer direct-llm-key",
|
||||||
wantCallCount: 1,
|
wantCallCount: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "environment key supplies authorization",
|
||||||
|
configureEnv: func(t *testing.T) {
|
||||||
|
t.Setenv("PROMPTKIT_TEST_API_KEY", " env-key ")
|
||||||
|
},
|
||||||
|
target: domain.ExecutionTarget{APIKeyEnv: "PROMPTKIT_TEST_API_KEY"},
|
||||||
|
wantAuthorization: "Bearer env-key",
|
||||||
|
wantCallCount: 1,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "no key omits authorization",
|
name: "no key omits authorization",
|
||||||
wantCallCount: 1,
|
wantCallCount: 1,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "missing environment key fails before transport",
|
name: "optional missing environment omits authorization",
|
||||||
configureEnv: func(t *testing.T) {
|
configureEnv: func(t *testing.T) {
|
||||||
t.Setenv("PROMPTKIT_MISSING_KEY", "")
|
t.Setenv("PROMPTKIT_MISSING_KEY", "")
|
||||||
},
|
},
|
||||||
target: domain.ExecutionTarget{APIKeyEnv: "PROMPTKIT_MISSING_KEY"},
|
target: domain.ExecutionTarget{APIKeyEnv: "PROMPTKIT_MISSING_KEY"},
|
||||||
wantErr: ErrInvalidRequest,
|
wantCallCount: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "required missing environment fails before transport",
|
||||||
|
configureEnv: func(t *testing.T) {
|
||||||
|
t.Setenv("PROMPTKIT_MISSING_KEY", "")
|
||||||
|
},
|
||||||
|
target: domain.ExecutionTarget{
|
||||||
|
APIKeyEnv: "PROMPTKIT_MISSING_KEY",
|
||||||
|
APIKeyRequired: true,
|
||||||
|
},
|
||||||
|
wantError: ErrInvalidRequest,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "required target without source fails before transport",
|
||||||
|
target: domain.ExecutionTarget{APIKeyRequired: true},
|
||||||
|
wantError: ErrInvalidRequest,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -545,9 +570,9 @@ func TestOpenAICompatibleClientAuthentication(t *testing.T) {
|
|||||||
request.Target = tc.target
|
request.Target = tc.target
|
||||||
|
|
||||||
_, err := client.Generate(context.Background(), request)
|
_, err := client.Generate(context.Background(), request)
|
||||||
if tc.wantErr != nil {
|
if tc.wantError != nil {
|
||||||
if !errors.Is(err, tc.wantErr) {
|
if !errors.Is(err, tc.wantError) {
|
||||||
t.Fatalf("error = %v, want %v", err, tc.wantErr)
|
t.Fatalf("error = %v, want %v", err, tc.wantError)
|
||||||
}
|
}
|
||||||
} else if err != nil {
|
} else if err != nil {
|
||||||
t.Fatalf("generate: %v", err)
|
t.Fatalf("generate: %v", err)
|
||||||
@@ -556,8 +581,13 @@ func TestOpenAICompatibleClientAuthentication(t *testing.T) {
|
|||||||
t.Fatalf("provider calls = %d, want %d", got, tc.wantCallCount)
|
t.Fatalf("provider calls = %d, want %d", got, tc.wantCallCount)
|
||||||
}
|
}
|
||||||
if tc.wantCallCount == 1 {
|
if tc.wantCallCount == 1 {
|
||||||
if got := provider.lastRequest(t).header.Get("Authorization"); got != tc.wantAuth {
|
values := provider.lastRequest(t).header.Values("Authorization")
|
||||||
t.Fatalf("Authorization = %q, want %q", got, tc.wantAuth)
|
if tc.wantAuthorization == "" {
|
||||||
|
if len(values) != 0 {
|
||||||
|
t.Fatalf("Authorization values = %q, want absent", values)
|
||||||
|
}
|
||||||
|
} else if len(values) != 1 || values[0] != tc.wantAuthorization {
|
||||||
|
t.Fatalf("Authorization values = %q, want [%q]", values, tc.wantAuthorization)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|||||||
@@ -3,7 +3,6 @@ package usecase
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"os"
|
|
||||||
"reflect"
|
"reflect"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
@@ -241,49 +240,76 @@ func excessivelyDeepPreparedJSONValue() any {
|
|||||||
return value
|
return value
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerRunPreparedRechecksEnvironmentCredentialBeforeAdmission(t *testing.T) {
|
func TestRunnerRunPreparedCredentialAvailabilityBeforeAdmission(t *testing.T) {
|
||||||
const environmentName = "PROMPTKIT_PREPARED_EXECUTION_TEST_KEY"
|
const environmentName = "PROMPTKIT_PREPARED_EXECUTION_TEST_KEY"
|
||||||
t.Setenv(environmentName, "available-during-preparation")
|
tests := []struct {
|
||||||
|
name string
|
||||||
profile := defaultExecutionProfile()
|
apiKeyRequired bool
|
||||||
profile.APIKeyEnv = environmentName
|
profileEnv bool
|
||||||
validator := &recordingValidationPreparer{plan: &recordingPreparedValidation{}}
|
overrideEnv bool
|
||||||
admitter := &fakeRunAdmitter{}
|
wantFailure bool
|
||||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "unexpected"}}
|
}{
|
||||||
runner := NewRunner(
|
{name: "optional environment becomes unavailable", profileEnv: true},
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
{
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
|
name: "required request environment becomes unavailable",
|
||||||
nil,
|
apiKeyRequired: true,
|
||||||
defaultArtifactReader(),
|
overrideEnv: true,
|
||||||
defaultRenderer(),
|
wantFailure: true,
|
||||||
llmClient,
|
},
|
||||||
validator,
|
|
||||||
admitter,
|
|
||||||
)
|
|
||||||
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
|
||||||
PromptID: "p",
|
|
||||||
ProfileID: "exec",
|
|
||||||
Inputs: singleInputRef(),
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("prepare execution: %v", err)
|
|
||||||
}
|
|
||||||
if err := os.Unsetenv(environmentName); err != nil {
|
|
||||||
t.Fatalf("unset credential environment: %v", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
result, err := runner.RunPrepared(context.Background(), prepared)
|
for _, tc := range tests {
|
||||||
if result != nil {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
t.Fatalf("credential failure returned partial result: %+v", result)
|
t.Setenv(environmentName, "available-during-preparation")
|
||||||
}
|
|
||||||
if !errors.Is(err, ErrInvalidRequest) || !errors.Is(err, ErrAPIKeyEnvMissing) {
|
profile := defaultExecutionProfile()
|
||||||
t.Fatalf("credential error identities are missing: %v", err)
|
profile.APIKeyRequired = tc.apiKeyRequired
|
||||||
}
|
if tc.profileEnv {
|
||||||
if len(admitter.backendIDs) != 0 || llmClient.calls != 0 {
|
profile.APIKeyEnv = environmentName
|
||||||
t.Fatalf("credential failure reached admission or generation: admission=%v generation=%d", admitter.backendIDs, llmClient.calls)
|
}
|
||||||
}
|
validator := &recordingValidationPreparer{plan: &recordingPreparedValidation{}}
|
||||||
if _, err := runner.RunPrepared(context.Background(), prepared); !errors.Is(err, ErrInvalidRequest) {
|
admitter := &fakeRunAdmitter{}
|
||||||
t.Fatalf("credential failure did not consume execution: %v", err)
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||||
|
runner := NewRunner(
|
||||||
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
|
||||||
|
nil,
|
||||||
|
defaultArtifactReader(),
|
||||||
|
defaultRenderer(),
|
||||||
|
llmClient,
|
||||||
|
validator,
|
||||||
|
admitter,
|
||||||
|
)
|
||||||
|
request := domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()}
|
||||||
|
if tc.overrideEnv {
|
||||||
|
request.Execution = &domain.ExecutionTargetOverride{APIKeyEnv: environmentName}
|
||||||
|
}
|
||||||
|
prepared, err := runner.PrepareExecution(context.Background(), request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution: %v", err)
|
||||||
|
}
|
||||||
|
t.Setenv(environmentName, "")
|
||||||
|
|
||||||
|
result, err := runner.RunPrepared(context.Background(), prepared)
|
||||||
|
if tc.wantFailure {
|
||||||
|
if result != nil || !errors.Is(err, ErrInvalidRequest) || !errors.Is(err, ErrAPIKeyEnvMissing) {
|
||||||
|
t.Fatalf("required credential result = (%+v, %v)", result, err)
|
||||||
|
}
|
||||||
|
if len(admitter.backendIDs) != 0 || llmClient.calls != 0 {
|
||||||
|
t.Fatalf("required credential reached admission or generation: admission=%v generation=%d", admitter.backendIDs, llmClient.calls)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if result == nil || err != nil {
|
||||||
|
t.Fatalf("optional credential result = (%+v, %v), want success", result, err)
|
||||||
|
}
|
||||||
|
if len(admitter.backendIDs) != 1 || llmClient.calls != 1 {
|
||||||
|
t.Fatalf("optional credential admission=%v generation=%d, want one each", admitter.backendIDs, llmClient.calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := runner.RunPrepared(context.Background(), prepared); !errors.Is(err, ErrInvalidRequest) {
|
||||||
|
t.Fatalf("execution outcome did not consume handle: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -624,12 +624,12 @@ func validateAPIKey(apiKeyEnv string, apiKey string, apiKeyRequired bool) error
|
|||||||
if strings.TrimSpace(apiKey) != "" {
|
if strings.TrimSpace(apiKey) != "" {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
if !apiKeyRequired {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
envName := strings.TrimSpace(apiKeyEnv)
|
envName := strings.TrimSpace(apiKeyEnv)
|
||||||
if envName == "" {
|
if envName == "" {
|
||||||
if apiKeyRequired {
|
return ErrAPIKeyRequired
|
||||||
return ErrAPIKeyRequired
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(os.Getenv(envName)) == "" {
|
if strings.TrimSpace(os.Getenv(envName)) == "" {
|
||||||
return fmt.Errorf("%w: api key environment variable %q is not set", ErrAPIKeyEnvMissing, envName)
|
return fmt.Errorf("%w: api key environment variable %q is not set", ErrAPIKeyEnvMissing, envName)
|
||||||
|
|||||||
@@ -1787,22 +1787,26 @@ func TestRunnerRunAPIKeyEnvResolvesFromEnvironment(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerRunAPIKeyEnvMissingEnvironmentValueFailsClearly(t *testing.T) {
|
func TestRunnerRunOptionalAPIKeyEnvMissingEnvironmentValueReachesLLM(t *testing.T) {
|
||||||
|
const environmentName = "PROMPTKIT_MISSING_KEY"
|
||||||
|
t.Setenv(environmentName, "")
|
||||||
|
|
||||||
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
||||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_MISSING_KEY"},
|
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: environmentName},
|
||||||
}}
|
}}
|
||||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil, nil)
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||||
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil)
|
||||||
|
|
||||||
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
result, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||||
if !errors.Is(err, ErrInvalidRequest) {
|
if err != nil || result == nil {
|
||||||
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
t.Fatalf("optional credential run = (%+v, %v), want success", result, err)
|
||||||
}
|
}
|
||||||
if !errors.Is(err, ErrAPIKeyEnvMissing) {
|
if llmClient.calls != 1 {
|
||||||
t.Fatalf("expected ErrAPIKeyEnvMissing, got %v", err)
|
t.Fatalf("LLM calls = %d, want 1", llmClient.calls)
|
||||||
}
|
}
|
||||||
if !strings.Contains(err.Error(), "PROMPTKIT_MISSING_KEY") {
|
if llmClient.lastReq.Target.APIKeyEnv != environmentName {
|
||||||
t.Fatalf("expected missing env name in error, got %v", err)
|
t.Fatalf("LLM api_key_env = %q, want %q", llmClient.lastReq.Target.APIKeyEnv, environmentName)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
|
||||||
"reflect"
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -524,19 +523,25 @@ func TestPreparedExecutionCredentialCapacityAndTimingBoundaries(t *testing.T) {
|
|||||||
engine, err := promptkit.NewEngine(
|
engine, err := promptkit.NewEngine(
|
||||||
promptkit.Config{},
|
promptkit.Config{},
|
||||||
promptkit.WithPromptFS(contractPromptFS("prepared", "profile", "content"), "."),
|
promptkit.WithPromptFS(contractPromptFS("prepared", "profile", "content"), "."),
|
||||||
promptkit.WithProfileFS(preparedCredentialProfileSource(environmentName), "."),
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "profile",
|
||||||
|
Endpoint: "http://example.test/v1",
|
||||||
|
Model: "model",
|
||||||
|
APIKeyRequired: true,
|
||||||
|
}),
|
||||||
promptkit.WithLLMClient(client),
|
promptkit.WithLLMClient(client),
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("construct credential engine: %v", err)
|
t.Fatalf("construct credential engine: %v", err)
|
||||||
}
|
}
|
||||||
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
|
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
||||||
|
PromptID: "prepared",
|
||||||
|
Execution: &promptkit.ExecutionTargetOverride{APIKeyEnv: environmentName},
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("prepare credential execution: %v", err)
|
t.Fatalf("prepare credential execution: %v", err)
|
||||||
}
|
}
|
||||||
if err := os.Unsetenv(environmentName); err != nil {
|
t.Setenv(environmentName, "")
|
||||||
t.Fatalf("unset credential environment: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
result, err := engine.RunPrepared(context.Background(), prepared)
|
result, err := engine.RunPrepared(context.Background(), prepared)
|
||||||
if result != nil ||
|
if result != nil ||
|
||||||
@@ -755,16 +760,6 @@ model: ` + model + `
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func preparedCredentialProfileSource(environmentName string) fstest.MapFS {
|
|
||||||
return fstest.MapFS{
|
|
||||||
"profile.yaml": &fstest.MapFile{Data: []byte(`id: profile
|
|
||||||
endpoint: http://example.test/v1
|
|
||||||
model: model
|
|
||||||
api_key_env: ` + environmentName + `
|
|
||||||
`)},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func preparedSchemaSource() fstest.MapFS {
|
func preparedSchemaSource() fstest.MapFS {
|
||||||
return fstest.MapFS{
|
return fstest.MapFS{
|
||||||
"schema.json": &fstest.MapFile{Data: []byte(`{
|
"schema.json": &fstest.MapFile{Data: []byte(`{
|
||||||
|
|||||||
15
types.go
15
types.go
@@ -322,7 +322,10 @@ type ExecutionTarget struct {
|
|||||||
// ReasoningEffort is the effective opaque provider-specific reasoning
|
// ReasoningEffort is the effective opaque provider-specific reasoning
|
||||||
// setting. An empty value instructs model clients to omit reasoning.
|
// setting. An empty value instructs model clients to omit reasoning.
|
||||||
ReasoningEffort string `json:"reasoning_effort"`
|
ReasoningEffort string `json:"reasoning_effort"`
|
||||||
// APIKeyEnv is an environment-variable name, not its credential value.
|
// APIKeyEnv is the resolved name of an optional environment lookup source,
|
||||||
|
// not its credential value. The built-in client omits Authorization when no
|
||||||
|
// usable direct or environment credential is available; injected clients may
|
||||||
|
// resolve this metadata differently.
|
||||||
APIKeyEnv string `json:"api_key_env"`
|
APIKeyEnv string `json:"api_key_env"`
|
||||||
// ExtraParams contains copied JSON-compatible provider parameters.
|
// ExtraParams contains copied JSON-compatible provider parameters.
|
||||||
ExtraParams map[string]any `json:"extra_params"`
|
ExtraParams map[string]any `json:"extra_params"`
|
||||||
@@ -426,9 +429,10 @@ type ExecutionTargetOverride struct {
|
|||||||
// inherited value and disables reasoning for this run. Non-blank values
|
// inherited value and disables reasoning for this run. Non-blank values
|
||||||
// are opaque and are not validated against a fixed vocabulary.
|
// are opaque and are not validated against a fixed vocabulary.
|
||||||
ReasoningEffort *string
|
ReasoningEffort *string
|
||||||
// APIKeyEnv replaces the profile or backend environment-variable name when
|
// APIKeyEnv replaces the profile or backend optional environment lookup
|
||||||
// non-blank. A direct RunRequest.APIKey still takes precedence over
|
// source when non-blank. A direct RunRequest.APIKey still takes precedence.
|
||||||
// environment lookup.
|
// The built-in client omits Authorization when neither source has a usable
|
||||||
|
// value; injected clients may resolve this metadata differently.
|
||||||
APIKeyEnv string
|
APIKeyEnv string
|
||||||
// ExtraParams, when non-empty, replaces the complete profile or backend map.
|
// ExtraParams, when non-empty, replaces the complete profile or backend map.
|
||||||
// Values must be JSON-compatible: nil, booleans, finite numbers, strings,
|
// Values must be JSON-compatible: nil, booleans, finite numbers, strings,
|
||||||
@@ -481,7 +485,8 @@ type Profile struct {
|
|||||||
ReasoningEffort string
|
ReasoningEffort string
|
||||||
// APIKeyRequired clears a backend's inherited API-key environment name and
|
// APIKeyRequired clears a backend's inherited API-key environment name and
|
||||||
// requires a non-blank RunRequest.APIKey unless the request explicitly
|
// requires a non-blank RunRequest.APIKey unless the request explicitly
|
||||||
// supplies ExecutionTargetOverride.APIKeyEnv. It does not store a credential.
|
// supplies ExecutionTargetOverride.APIKeyEnv. When false, a named
|
||||||
|
// environment source remains optional. It does not store a credential.
|
||||||
APIKeyRequired bool
|
APIKeyRequired bool
|
||||||
// ExtraParams contains provider-specific JSON-compatible values. An empty
|
// ExtraParams contains provider-specific JSON-compatible values. An empty
|
||||||
// map inherits backend request defaults, when any. WithProfiles validates
|
// map inherits backend request defaults, when any. WithProfiles validates
|
||||||
|
|||||||
Reference in New Issue
Block a user