Compare commits
19 Commits
c239304c2a
...
v0.8.0
| Author | SHA1 | Date | |
|---|---|---|---|
| 2e76003fd5 | |||
| 36ce5a5099 | |||
| 465dc1389d | |||
| ae6f1a9865 | |||
| ee99dc9478 | |||
| 00ee5893e9 | |||
| 64d1cffd89 | |||
| f9e8afa2c3 | |||
| e827631d8c | |||
| 115fe8ba58 | |||
| c53250f023 | |||
| 3d99483219 | |||
| 67f788b1e2 | |||
| 764103a2e2 | |||
| a08dd83d1f | |||
| e8922d8ec5 | |||
| 2d44305a8a | |||
| a11c80291e | |||
| 3239567297 |
10
README.md
10
README.md
@@ -33,10 +33,16 @@ boundary and constraints that framework work must preserve.
|
|||||||
|
|
||||||
## Release Guidance
|
## Release Guidance
|
||||||
|
|
||||||
Consumers upgrading from `v0.5.0` to `v0.6.0` should read the
|
Consumers upgrading from `v0.7.0` to `v0.8.0` should read the
|
||||||
[v0.6.0 changelog and migration guide](docs/releases/v0.6.0.md).
|
[v0.8.0 changelog and migration guide](docs/releases/v0.8.0.md).
|
||||||
|
|
||||||
Earlier adopters can consult the
|
Earlier adopters can consult the
|
||||||
|
[v0.7.0 changelog and migration guide](docs/releases/v0.7.0.md).
|
||||||
|
|
||||||
|
Consumers upgrading from `v0.5.0` to `v0.6.0` can consult the
|
||||||
|
[v0.6.0 changelog and migration guide](docs/releases/v0.6.0.md).
|
||||||
|
|
||||||
|
Consumers upgrading from `v0.4.0` to `v0.5.0` can consult the
|
||||||
[v0.5.0 changelog and migration guide](docs/releases/v0.5.0.md).
|
[v0.5.0 changelog and migration guide](docs/releases/v0.5.0.md).
|
||||||
|
|
||||||
Consumers upgrading from `v0.3.0` to `v0.4.0` should read the
|
Consumers upgrading from `v0.3.0` to `v0.4.0` should read the
|
||||||
|
|||||||
@@ -30,9 +30,12 @@ type Backend struct {
|
|||||||
// requires an absolute HTTP or HTTPS URL with a host and without user
|
// requires an absolute HTTP or HTTPS URL with a host and without user
|
||||||
// information, a query string, or a fragment. Paths are allowed.
|
// information, a query string, or a fragment. Paths are allowed.
|
||||||
Endpoint string
|
Endpoint string
|
||||||
// APIKeyEnv optionally names the environment variable containing the API
|
// APIKeyEnv optionally names an environment lookup source for an API key.
|
||||||
// key. NewEngine trims it and requires the portable form
|
// NewEngine trims it and requires the portable form [A-Za-z_][A-Za-z0-9_]*.
|
||||||
// [A-Za-z_][A-Za-z0-9_]*. Store only the name, never a credential value.
|
// A direct RunRequest.APIKey takes precedence. When no usable credential is
|
||||||
|
// available, the built-in client omits Authorization; injected clients own
|
||||||
|
// their own credential-resolution behavior. Store only the name, never a
|
||||||
|
// credential value.
|
||||||
APIKeyEnv string
|
APIKeyEnv string
|
||||||
// ExtraParams contains backend-wide request defaults. Values must be
|
// ExtraParams contains backend-wide request defaults. Values must be
|
||||||
// JSON-compatible, finite, acyclic, and keyed by non-empty strings. Keys
|
// JSON-compatible, finite, acyclic, and keyed by non-empty strings. Keys
|
||||||
|
|||||||
@@ -173,6 +173,29 @@ semantics. The
|
|||||||
[OpenAI-compatible integration contract](../integrations/openai-compatible-chat.md)
|
[OpenAI-compatible integration contract](../integrations/openai-compatible-chat.md)
|
||||||
owns the built-in client's outbound HTTP behavior.
|
owns the built-in client's outbound HTTP behavior.
|
||||||
|
|
||||||
|
### Repair A Structured Result
|
||||||
|
|
||||||
|
Set a small additional-call budget when a structurally invalid result can be
|
||||||
|
corrected automatically:
|
||||||
|
|
||||||
|
```go
|
||||||
|
request.Validation = &promptkit.OutputContract{
|
||||||
|
Format: promptkit.FormatJSON,
|
||||||
|
ValidationMode: promptkit.ValidationJSONSchema,
|
||||||
|
SchemaPath: "events.schema.json",
|
||||||
|
RepairAttempts: 1,
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Each repair attempt is another model call, so it can increase latency and
|
||||||
|
usage; `RunResult.Usage` is cumulative and `Validation.RepairAttempts` reports
|
||||||
|
calls actually started. Exhaustion still returns the final failed validation
|
||||||
|
result. `basic` validation can also repair an empty candidate, but structural
|
||||||
|
validity is not evidence of factual or domain correctness. See the
|
||||||
|
[output-contract format reference](../formats.md#output-contract) and
|
||||||
|
[`OutputContract` GoDoc](../../types.go) for the exact budget and eligibility
|
||||||
|
rules.
|
||||||
|
|
||||||
## Inputs, Profiles, And Overrides
|
## Inputs, Profiles, And Overrides
|
||||||
|
|
||||||
Use `File`, `Inline`, or `InlineWithURI` to construct request inputs. A request
|
Use `File`, `Inline`, or `InlineWithURI` to construct request inputs. A request
|
||||||
@@ -189,6 +212,26 @@ For programmatic profiles,
|
|||||||
[`OpenAICompatibleProfile`](../../profiles.go) converts ordinary
|
[`OpenAICompatibleProfile`](../../profiles.go) converts ordinary
|
||||||
OpenAI-compatible settings into a value accepted by `WithProfiles`.
|
OpenAI-compatible settings into a value accepted by `WithProfiles`.
|
||||||
|
|
||||||
|
### Alias A Built-In Profile
|
||||||
|
|
||||||
|
Give an application-owned profile ID a built-in base when prompts should select
|
||||||
|
the application ID while inheriting the built-in target. The child can override
|
||||||
|
only the setting it owns:
|
||||||
|
|
||||||
|
```go
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "weather-light",
|
||||||
|
BaseProfileID: "deepseek-4-flash",
|
||||||
|
ReasoningEffort: "high",
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
Select `weather-light` in a prompt or `RunRequest.ProfileID`; it remains the
|
||||||
|
reported selected profile. See the [profile inheritance format
|
||||||
|
reference](../formats.md#profile-inheritance) and the
|
||||||
|
[`Profile` GoDoc](../../types.go) for exact lookup, merging, and validation
|
||||||
|
behavior.
|
||||||
|
|
||||||
### Use The Rakestrawhome Built-In Profile
|
### Use The Rakestrawhome Built-In Profile
|
||||||
|
|
||||||
Set `RAKESTRAWHOME_INFERENCE_API_KEY` in the application environment, then
|
Set `RAKESTRAWHOME_INFERENCE_API_KEY` in the application environment, then
|
||||||
@@ -231,7 +274,7 @@ if err != nil {
|
|||||||
|
|
||||||
target := inspection.EffectiveModelParams
|
target := inspection.EffectiveModelParams
|
||||||
if target.APIKeyEnv != "" {
|
if target.APIKeyEnv != "" {
|
||||||
// Apply application policy for the named environment variable.
|
// This is a configured optional environment lookup source.
|
||||||
} else if inspection.APIKeyRequired {
|
} else if inspection.APIKeyRequired {
|
||||||
// Arrange a direct credential before later execution.
|
// Arrange a direct credential before later execution.
|
||||||
}
|
}
|
||||||
@@ -240,9 +283,11 @@ if target.APIKeyEnv != "" {
|
|||||||
Use this configuration-time boundary when only the profile and its target need
|
Use this configuration-time boundary when only the profile and its target need
|
||||||
checking. Use `Prepare` when the application also needs prompt, input, schema,
|
checking. Use `Prepare` when the application also needs prompt, input, schema,
|
||||||
or rendering work; use prepared execution when that work must remain tied to a
|
or rendering work; use prepared execution when that work must remain tied to a
|
||||||
later execution. Inspection reports credential requirements but leaves the
|
later execution. A reported `APIKeyEnv` is a configured optional source, while
|
||||||
timing of credential enforcement to the application. The method's
|
`APIKeyRequired` is the explicit local requirement. The
|
||||||
[GoDoc](../../engine.go) owns its exact result and error contract.
|
[credential format reference](../formats.md#credentials) and the method's
|
||||||
|
[GoDoc](../../engine.go) own the exact precedence, timing, result, and error
|
||||||
|
contracts.
|
||||||
|
|
||||||
### Set A Per-Run Session And Reasoning
|
### Set A Per-Run Session And Reasoning
|
||||||
|
|
||||||
|
|||||||
@@ -126,7 +126,7 @@ outbound integration determines its wire representation.
|
|||||||
| `format` | yes | `text`, `markdown`, or `json`. |
|
| `format` | yes | `text`, `markdown`, or `json`. |
|
||||||
| `validation_mode` | yes | `none`, `basic`, `json`, or `json_schema`. |
|
| `validation_mode` | yes | `none`, `basic`, `json`, or `json_schema`. |
|
||||||
| `schema_path` | for `json_schema` | Path to a schema in the configured schema source. |
|
| `schema_path` | for `json_schema` | Path to a schema in the configured schema source. |
|
||||||
| `repair_attempts` | no | Integer zero or greater; omitted means zero. |
|
| `repair_attempts` | no | Integer from zero through three; omitted means zero. A positive value requires `basic`, `json`, or `json_schema` validation. |
|
||||||
|
|
||||||
The validation modes behave as follows:
|
The validation modes behave as follows:
|
||||||
|
|
||||||
@@ -136,9 +136,15 @@ The validation modes behave as follows:
|
|||||||
- `json_schema` requires valid JSON that satisfies the selected schema.
|
- `json_schema` requires valid JSON that satisfies the selected schema.
|
||||||
|
|
||||||
`format` controls output artifact metadata. JSON Schema mode also supplies the
|
`format` controls output artifact metadata. JSON Schema mode also supplies the
|
||||||
schema to compatible model clients as structured-output metadata. The public
|
schema to compatible model clients as structured-output metadata. Plain `json`
|
||||||
engine does not install an output repairer, so its validation is single-pass
|
validation accepts every valid JSON value and does not request a provider-native
|
||||||
even when a positive `repair_attempts` value is present.
|
JSON-object constraint.
|
||||||
|
|
||||||
|
`repair_attempts` counts additional generation calls after a failed validation.
|
||||||
|
Zero is single-pass. With a positive eligible budget, Promptkit stops at the
|
||||||
|
first valid candidate. If the budget is exhausted, it returns the final
|
||||||
|
candidate and its complete failed validation result; generation and operational
|
||||||
|
validation failures remain errors. `none` never permits repair.
|
||||||
|
|
||||||
A request-level `OutputContract` replaces the complete prompt output contract.
|
A request-level `OutputContract` replaces the complete prompt output contract.
|
||||||
It does not merge individual fields. If its format is empty, Promptkit uses
|
It does not merge individual fields. If its format is empty, Promptkit uses
|
||||||
@@ -173,9 +179,19 @@ extra_params:
|
|||||||
provider_option: enabled
|
provider_option: enabled
|
||||||
```
|
```
|
||||||
|
|
||||||
|
A derived profile can use a named base and override only the settings it owns:
|
||||||
|
|
||||||
|
```yaml
|
||||||
|
id: local-summary-fast
|
||||||
|
base_profile: local-summary
|
||||||
|
timeout_seconds: 30
|
||||||
|
reasoning_effort: low
|
||||||
|
```
|
||||||
|
|
||||||
| Field | Required | Meaning |
|
| Field | Required | Meaning |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| `id` | yes | Profile identifier, trimmed before selection and publication. It must be non-empty after trimming and unique within one source after normalization. |
|
| `id` | yes | Profile identifier, trimmed before selection and publication. It must be non-empty after trimming and unique within one source after normalization. |
|
||||||
|
| `base_profile` | no | One optional parent profile ID. A derived profile may inherit target fields from it. |
|
||||||
| `backend` | unless `endpoint` is present | Backend registry ID. It is trimmed and registry membership is checked when the profile is prepared or inspected. |
|
| `backend` | unless `endpoint` is present | Backend registry ID. It is trimmed and registry membership is checked when the profile is prepared or inspected. |
|
||||||
| `endpoint` | unless `backend` is present | OpenAI-compatible base URL, including an API version path when required. A nonempty value is trimmed and must be absolute HTTP or HTTPS with a host and without user information, a query, or a fragment. When both connection fields are present, this overrides the backend endpoint without changing backend identity. |
|
| `endpoint` | unless `backend` is present | OpenAI-compatible base URL, including an API version path when required. A nonempty value is trimmed and must be absolute HTTP or HTTPS with a host and without user information, a query, or a fragment. When both connection fields are present, this overrides the backend endpoint without changing backend identity. |
|
||||||
| `model` | yes | Non-empty provider model name. |
|
| `model` | yes | Non-empty provider model name. |
|
||||||
@@ -185,12 +201,17 @@ extra_params:
|
|||||||
| `timeout_seconds` | no | Per-generation deadline in whole seconds; integer zero or greater. |
|
| `timeout_seconds` | no | Per-generation deadline in whole seconds; integer zero or greater. |
|
||||||
| `service_tier` | no | Provider-specific request tier. |
|
| `service_tier` | no | Provider-specific request tier. |
|
||||||
| `reasoning_effort` | no | Provider-specific reasoning setting. |
|
| `reasoning_effort` | no | Provider-specific reasoning setting. |
|
||||||
| `api_key_env` | no | Name of an environment variable containing the API key. |
|
| `api_key_env` | no | Optional environment-variable lookup source for an API key. |
|
||||||
| `extra_params` | no | JSON-compatible provider-specific outbound fields. |
|
| `extra_params` | no | JSON-compatible provider-specific outbound fields. |
|
||||||
|
|
||||||
Raw `api_key` is prohibited in profile YAML. Store only an environment
|
Raw `api_key` is prohibited in profile YAML. Store only an environment
|
||||||
variable name in `api_key_env`.
|
variable name in `api_key_env`.
|
||||||
|
|
||||||
|
A standalone profile must provide a model and at least one of `backend` or
|
||||||
|
`endpoint`. A derived profile may omit those target fields because its selected
|
||||||
|
base chain can provide them. Local parsing still validates a derived profile's
|
||||||
|
own ID, supplied endpoint, execution-setting bounds, and `extra_params`.
|
||||||
|
|
||||||
Promptkit does not infer a backend from a model or endpoint. Endpoint-only
|
Promptkit does not infer a backend from a model or endpoint. Endpoint-only
|
||||||
profiles remain supported and have no effective backend ID.
|
profiles remain supported and have no effective backend ID.
|
||||||
The engine always provides the built-in `openrouter` and `rakestrawhome` IDs.
|
The engine always provides the built-in `openrouter` and `rakestrawhome` IDs.
|
||||||
@@ -266,6 +287,32 @@ profiles. They use `APIKeyRequired` for request-scoped credentials instead of
|
|||||||
`api_key_env`. Preparation and exact profile inspection use this same source
|
`api_key_env`. Preparation and exact profile inspection use this same source
|
||||||
precedence.
|
precedence.
|
||||||
|
|
||||||
|
When a selected definition names `base_profile`, every profile ID in that
|
||||||
|
chain is looked up through this same precedence order. A higher-precedence
|
||||||
|
definition therefore shadows a lower-precedence definition of the same base
|
||||||
|
ID, including a built-in. References are not source-qualified.
|
||||||
|
|
||||||
|
### Profile Inheritance
|
||||||
|
|
||||||
|
Promptkit resolves one linear base chain of at most 32 profiles, including the
|
||||||
|
selected profile. It merges settings from the root base to the selected leaf.
|
||||||
|
The leaf's `id` remains the selected profile identity. Nonblank string fields
|
||||||
|
(`backend`, `endpoint`, `model`, `service_tier`, `reasoning_effort`, and
|
||||||
|
`api_key_env`) and nonzero numeric fields replace inherited values. A nonempty
|
||||||
|
`extra_params` map replaces the complete inherited map rather than merging
|
||||||
|
keys, and `APIKeyRequired: true` remains true through the chain. Backend and
|
||||||
|
endpoint are independent: replacing one does not clear the other.
|
||||||
|
|
||||||
|
There is no profile-level clearing syntax. Blank strings, zero numbers, false,
|
||||||
|
and empty maps remain unspecified and inherit from a base. Use existing
|
||||||
|
presence-aware request overrides where an execution needs an explicit zero or
|
||||||
|
empty reasoning setting.
|
||||||
|
|
||||||
|
An absent directly selected profile reports the ordinary not-found error. Once
|
||||||
|
the selected profile exists, a missing base, cycle, overlong chain, or
|
||||||
|
incomplete resolved target is a profile-load failure. Ordinary operations
|
||||||
|
resolve chains afresh; prepared execution retains the fully resolved target.
|
||||||
|
|
||||||
## Built-In Profile Catalog
|
## Built-In Profile Catalog
|
||||||
|
|
||||||
Every built-in profile selects one maintained built-in backend and inherits
|
Every built-in profile selects one maintained built-in backend and inherits
|
||||||
@@ -318,16 +365,25 @@ schema produces a failed validation result.
|
|||||||
Credential values belong at the request or environment boundary, never in
|
Credential values belong at the request or environment boundary, never in
|
||||||
prompt, profile, schema, or example files:
|
prompt, profile, schema, or example files:
|
||||||
|
|
||||||
- a file profile names an environment variable with `api_key_env`;
|
- a backend or file profile can name an optional environment lookup source
|
||||||
- an in-memory profile may set `APIKeyRequired`;
|
with `APIKeyEnv` or `api_key_env`;
|
||||||
- a request can provide a direct `APIKey` or override `APIKeyEnv`; and
|
- an in-memory profile may set `APIKeyRequired` as an explicit local
|
||||||
|
requirement;
|
||||||
|
- a request can provide a direct `APIKey` or override the optional `APIKeyEnv`
|
||||||
|
source; and
|
||||||
- a direct request key takes precedence over environment lookup.
|
- a direct request key takes precedence over environment lookup.
|
||||||
|
|
||||||
After a direct request key, the credential-source precedence is request
|
After a direct request key, the credential-source precedence is request
|
||||||
`APIKeyEnv`, profile `api_key_env`, then the backend default. An in-memory
|
`APIKeyEnv`, profile `api_key_env`, then the backend default. An in-memory
|
||||||
profile with `APIKeyRequired` clears an inherited backend environment name and
|
profile with `APIKeyRequired` clears an inherited backend environment name and
|
||||||
requires a direct key unless the request explicitly supplies `APIKeyEnv`.
|
requires a direct key unless the request explicitly supplies `APIKeyEnv`.
|
||||||
Promptkit validates required credential availability during preparation.
|
Named environment sources are optional: when the selected source is absent,
|
||||||
|
empty, or whitespace-only, the built-in client omits the `Authorization`
|
||||||
|
header and handles the provider response normally. `APIKeyRequired` is the
|
||||||
|
only explicit local availability requirement. Promptkit validates required
|
||||||
|
credential availability during preparation and rechecks it when a prepared
|
||||||
|
execution runs. Injected clients receive resolved source metadata but define
|
||||||
|
their own credential-resolution behavior.
|
||||||
Direct keys are excluded from JSON results and redacted by public string
|
Direct keys are excluded from JSON results and redacted by public string
|
||||||
formatters. Environment-variable names may appear in prepared metadata, but
|
formatters. Environment-variable names may appear in prepared metadata, but
|
||||||
their values do not.
|
their values do not.
|
||||||
|
|||||||
@@ -31,11 +31,13 @@ does not serialize it in the provider request.
|
|||||||
|
|
||||||
## Authentication
|
## Authentication
|
||||||
|
|
||||||
A non-empty API key supplied directly on the execution target takes
|
A usable API key supplied directly on the execution target takes precedence.
|
||||||
precedence. Otherwise, when an API-key environment-variable name is supplied,
|
Otherwise, when an API-key environment-variable name is supplied, the client
|
||||||
the client reads that variable and requires a non-empty value. The selected
|
reads and trims that variable. A bearer header is sent only when the resolved
|
||||||
key is sent as `Authorization: Bearer <key>`. No authorization header is sent
|
direct or environment credential is non-empty. When neither source is usable,
|
||||||
when neither mechanism is configured.
|
the client omits `Authorization` and handles the provider response normally.
|
||||||
|
An explicitly required target with no usable source is rejected before
|
||||||
|
transport.
|
||||||
|
|
||||||
The target contains the already resolved environment-variable name: an
|
The target contains the already resolved environment-variable name: an
|
||||||
explicit request override takes precedence over profile metadata, which takes
|
explicit request override takes precedence over profile metadata, which takes
|
||||||
@@ -65,7 +67,8 @@ The client conditionally includes:
|
|||||||
- non-empty `service_tier` and effective `reasoning_effort`; an explicitly
|
- non-empty `service_tier` and effective `reasoning_effort`; an explicitly
|
||||||
disabled reasoning setting is empty and therefore omitted; and
|
disabled reasoning setting is empty and therefore omitted; and
|
||||||
- `response_format` for JSON Schema structured output, including its name,
|
- `response_format` for JSON Schema structured output, including its name,
|
||||||
strict flag, and schema document.
|
strict flag, and schema document. Plain JSON validation does not add an
|
||||||
|
object-only response constraint.
|
||||||
|
|
||||||
The engine resolves backend, profile, and request extra-parameter maps by
|
The engine resolves backend, profile, and request extra-parameter maps by
|
||||||
whole-map replacement rather than key merging. The resulting effective map is
|
whole-map replacement rather than key merging. The resulting effective map is
|
||||||
@@ -97,10 +100,12 @@ drained.
|
|||||||
|
|
||||||
The bounded body must contain exactly one OpenAI-compatible JSON response
|
The bounded body must contain exactly one OpenAI-compatible JSON response
|
||||||
object followed only by JSON whitespace and EOF. The client returns the first
|
object followed only by JSON whitespace and EOF. The client returns the first
|
||||||
choice's non-empty message content and maps prompt, completion, total, cached,
|
choice's explicitly present string message content, including an empty or
|
||||||
and cache-write token counts. Invalid or truncated JSON, trailing non-whitespace
|
whitespace-only string, and maps prompt, completion, total, cached, and
|
||||||
data, a second JSON value, absent choices, empty first-choice content, and size
|
cache-write token counts. Invalid or truncated JSON, trailing non-whitespace
|
||||||
overflow are malformed responses and return no partial result.
|
data, a second JSON value, absent choices, missing content, `null` content,
|
||||||
|
non-string content, and size overflow are malformed responses and return no
|
||||||
|
partial result.
|
||||||
|
|
||||||
For a non-2xx status, Promptkit recognizes one JSON document with a top-level
|
For a non-2xx status, Promptkit recognizes one JSON document with a top-level
|
||||||
object-valued `error` member. Its optional `message` and `type` fields must be
|
object-valued `error` member. Its optional `message` and `type` fields must be
|
||||||
|
|||||||
@@ -55,12 +55,14 @@ the target, rendered messages, and structured-output constraint retained by
|
|||||||
executable preparation. Execution does not reopen or rerender consumer
|
executable preparation. Execution does not reopen or rerender consumer
|
||||||
sources.
|
sources.
|
||||||
|
|
||||||
Before backend admission, the runner rechecks that the frozen credential
|
Before backend admission, the runner rechecks a frozen credential
|
||||||
environment-variable name is available. The handle does not retain the
|
environment-variable name only when the target explicitly requires a
|
||||||
environment value; the model client resolves the value visible when generation
|
credential. The handle does not retain the environment value; the model client
|
||||||
begins. A direct request key remains in private execution state only until the
|
resolves the value visible when generation begins. For optional sources with no
|
||||||
claimed execution finishes or an unclaimed handle is discarded. Exact public
|
usable value, the built-in client omits `Authorization` and continues to the
|
||||||
ownership and redaction semantics belong to the
|
provider. A direct request key remains in private execution state only until
|
||||||
|
the claimed execution finishes or an unclaimed handle is discarded. Exact
|
||||||
|
public ownership and redaction semantics belong to the
|
||||||
[`PreparedExecution` GoDoc](../../prepared_execution.go).
|
[`PreparedExecution` GoDoc](../../prepared_execution.go).
|
||||||
|
|
||||||
## Failure Categories
|
## Failure Categories
|
||||||
@@ -68,12 +70,18 @@ ownership and redaction semantics belong to the
|
|||||||
The package preserves distinct error identities for invalid client
|
The package preserves distinct error identities for invalid client
|
||||||
configuration, invalid generation requests, request execution failures,
|
configuration, invalid generation requests, request execution failures,
|
||||||
non-success provider statuses, and malformed successful responses. Provider
|
non-success provider statuses, and malformed successful responses. Provider
|
||||||
response bodies are not included in non-success errors.
|
response bodies are never exposed in raw form through non-success errors.
|
||||||
|
|
||||||
Invalid nonempty configured endpoints are configuration failures. A missing or
|
Invalid nonempty configured endpoints are configuration failures. A missing or
|
||||||
invalid final selected endpoint is an invalid generation request and is
|
invalid final selected endpoint is an invalid generation request and is
|
||||||
rejected before transport.
|
rejected before transport.
|
||||||
|
|
||||||
|
Authentication resolves a trimmed direct key before a trimmed configured
|
||||||
|
environment value. Optional missing, empty, or whitespace-only sources do not
|
||||||
|
block transport and produce no `Authorization` header. An explicitly required
|
||||||
|
target with no usable source is rejected before transport with the existing
|
||||||
|
invalid-request diagnostics.
|
||||||
|
|
||||||
Successful response bodies have a fixed 16 MiB limit enforced by declared
|
Successful response bodies have a fixed 16 MiB limit enforced by declared
|
||||||
length and by reading at most one byte beyond the boundary. The decoder accepts
|
length and by reading at most one byte beyond the boundary. The decoder accepts
|
||||||
exactly one JSON object plus trailing whitespace and EOF. Size overflow,
|
exactly one JSON object plus trailing whitespace and EOF. Size overflow,
|
||||||
@@ -81,6 +89,12 @@ truncation, malformed JSON, trailing data, and a second value are malformed
|
|||||||
responses with no partial result or provider content in the error. Every body
|
responses with no partial result or provider content in the error. Every body
|
||||||
is closed, and an unbounded oversized stream is not drained.
|
is closed, and an unbounded oversized stream is not drained.
|
||||||
|
|
||||||
|
After framing succeeds, the first choice must contain an explicitly present
|
||||||
|
string `message.content`. The string is returned exactly, including empty or
|
||||||
|
whitespace-only content. Missing choices, missing or `null` content, and
|
||||||
|
non-string content are malformed responses. Output validation and correction
|
||||||
|
eligibility remain outside this package.
|
||||||
|
|
||||||
For a non-success response, `ProviderHTTPError` retains the HTTP status and
|
For a non-success response, `ProviderHTTPError` retains the HTTP status and
|
||||||
only normalized detail from the bounded recognized envelope. It retains
|
only normalized detail from the bounded recognized envelope. It retains
|
||||||
`ErrUnexpectedStatus` through unwrapping. The client owns response closure;
|
`ErrUnexpectedStatus` through unwrapping. The client owns response closure;
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ contributor workflow and validation.
|
|||||||
|
|
||||||
| Component | Implemented responsibility | References |
|
| Component | Implemented responsibility | References |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request, result, prompt-inspection, and profile-inspection values, opaque prepared-execution handles, profile construction, extension interfaces, value conversion, redacted formatting, typed capacity and generation error mapping, and engine-local profile-source assembly including application fallbacks. | [Package GoDoc](../../doc.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine assembly](../../engine.go) |
|
| Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request, result, prompt-inspection, and profile-inspection values, opaque prepared-execution handles, profile construction, extension interfaces, value conversion, redacted formatting, typed capacity and generation error mapping, engine-local profile-source assembly including application fallbacks, and bounded output-repair assembly. | [Package GoDoc](../../doc.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine assembly](../../engine.go) |
|
||||||
| `examples/go-library/prepare` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, and `Prepare`. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
|
| `examples/go-library/prepare` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, and `Prepare`. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
|
||||||
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
||||||
| `internal/backend` | Constructs each engine's immutable registry from the maintained built-in definitions and consumer additions, validates and defensively copies definitions through the shared JSON-value package, and consumes the LLM-owned OpenAI-compatible reserved request-field rule. | [Backend registry](../../internal/backend/registry.go) |
|
| `internal/backend` | Constructs each engine's immutable registry from the maintained built-in definitions and consumer additions, validates and defensively copies definitions through the shared JSON-value package, and consumes the LLM-owned OpenAI-compatible reserved request-field rule. | [Backend registry](../../internal/backend/registry.go) |
|
||||||
@@ -21,13 +21,13 @@ contributor workflow and validation.
|
|||||||
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
||||||
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types and rejecting cycles or excessive depth and work. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types and rejecting cycles or excessive depth and work. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
||||||
| `internal/promptdef` | Loads strictly decoded, validated prompt definitions from filesystem and `fs.FS` sources, including version selection and contained file-backed message content. | [Framework formats](../formats.md), [prompt-definition repository](../../internal/promptdef/filesystem_repository.go) |
|
| `internal/promptdef` | Loads strictly decoded, validated prompt definitions from filesystem and `fs.FS` sources, including version selection and contained file-backed message content. | [Framework formats](../formats.md), [prompt-definition repository](../../internal/promptdef/filesystem_repository.go) |
|
||||||
| `internal/profile` | Loads strictly decoded, validated execution profiles, including backend selection, from filesystem and `fs.FS` sources and composes repositories with error-preserving fallback. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go) |
|
| `internal/profile` | Loads strictly decoded, locally validated execution profiles from filesystem and `fs.FS` sources, overlays raw sources with error-preserving fallback, and resolves inherited profiles. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go), [internal sources](sources.md#profiles-and-built-ins) |
|
||||||
| `internal/profile/builtin` | Embeds the built-in profile catalog, whose entries select maintained built-in backends. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) |
|
| `internal/profile/builtin` | Embeds the built-in profile catalog, whose entries select maintained built-in backends. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) |
|
||||||
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
|
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
|
||||||
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
||||||
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates operation-local validation plans with canonical contained schema resources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates operation-local validation plans with canonical contained schema resources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
||||||
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded structured non-success response decoding, successful-response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded structured non-success response decoding, successful-response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
||||||
| `internal/usecase` | Resolves prompt definitions and hashes, profiles, backends, and targets for exact inspection and request settings for preparation, and coordinates ordinary execution and one-attempt prepared execution across internal sources, rendering, artifact loading, operation-local validation plans, generation, capacity, and optional repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
| `internal/usecase` | Resolves prompt definitions and hashes, profiles, backends, and targets for exact inspection and request settings for preparation, and coordinates ordinary execution and one-attempt prepared execution across internal sources, rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
||||||
|
|
||||||
The root package assembles these internal components without exposing their
|
The root package assembles these internal components without exposing their
|
||||||
representations. Consumers depend only on the root facade.
|
representations. Consumers depend only on the root facade.
|
||||||
|
|||||||
@@ -23,8 +23,10 @@ built-in backend and validated consumer additions, one engine-local run
|
|||||||
admitter, and a model client wrapped by the same capacity manager. Validation
|
admitter, and a model client wrapped by the same capacity manager. Validation
|
||||||
plans and provider-facing schema metadata come from the validator's preparation
|
plans and provider-facing schema metadata come from the validator's preparation
|
||||||
interface.
|
interface.
|
||||||
An output repairer can be injected internally, but the ordinary runner
|
The root engine supplies one default output repairer through the explicit
|
||||||
constructor does not enable one.
|
runner constructor, using the same capacity-wrapped client as initial
|
||||||
|
generation. The no-repair runner constructor remains available for focused
|
||||||
|
internal callers and tests.
|
||||||
|
|
||||||
Each invocation carries its state in request, prepared-run, and result values.
|
Each invocation carries its state in request, prepared-run, and result values.
|
||||||
The runner has no durable run or session store.
|
The runner has no durable run or session store.
|
||||||
@@ -125,8 +127,8 @@ admitter is an internal unlimited fallback. After successful admission, `Run`
|
|||||||
immediately defers the returned release function, performs the completion
|
immediately defers the returned release function, performs the completion
|
||||||
phase, makes one initial generation call, builds the named output artifact,
|
phase, makes one initial generation call, builds the named output artifact,
|
||||||
and validates that artifact with the plan compiled during completion. Invalid
|
and validates that artifact with the plan compiled during completion. Invalid
|
||||||
generated content remains a validation result; an inability to perform
|
generated content remains a validation result; an inability to generate or
|
||||||
validation is an operational error.
|
perform validation is an operational error.
|
||||||
|
|
||||||
Validation preparation and execution honor cancellation at every
|
Validation preparation and execution honor cancellation at every
|
||||||
Promptkit-controlled boundary and do not publish a partial plan or result.
|
Promptkit-controlled boundary and do not publish a partial plan or result.
|
||||||
@@ -142,17 +144,25 @@ serializing preparation or validation behind the active-generation limit.
|
|||||||
The wrapped model client separately acquires a FIFO active permit only around
|
The wrapped model client separately acquires a FIFO active permit only around
|
||||||
each actual generation call.
|
each actual generation call.
|
||||||
|
|
||||||
When an internal repairer is present, a JSON or JSON Schema content failure can
|
After a failed `basic`, JSON, or JSON Schema validation with a positive frozen
|
||||||
trigger bounded repair attempts. Repair receives the effective execution
|
budget, the installed repairer can make a bounded corrective call. Each request
|
||||||
target, explicit numeric-presence bits, credential, backend identity, session
|
starts with a fresh copy of the complete original rendered messages, includes
|
||||||
ID, validation errors, prior output, and structured-output specification. One
|
only the latest nonempty candidate as an assistant message, and appends one
|
||||||
request constructor supplies those common fields to initial and repair
|
corrective user message. Empty candidates omit that assistant message. The
|
||||||
generation while their rendered prompts remain intentionally distinct. The
|
correction carries validation diagnostics as JSON data bounded to 64 KiB; the
|
||||||
default repairer uses the same wrapped client as initial generation, so each
|
full diagnostics remain in the validation result.
|
||||||
repair reacquires the selected backend's active permit while remaining inside
|
|
||||||
its original admission lease. Repair never performs a second bounded
|
Repair receives the effective execution target, explicit numeric-presence bits,
|
||||||
admission, and repaired outputs use the operation's existing validation plan.
|
credential, backend identity, session ID, and structured-output specification.
|
||||||
This capability remains internal and is not a public option.
|
The same request constructor supplies those common fields to initial and repair
|
||||||
|
generation. The default repairer uses the same wrapped client as initial
|
||||||
|
generation, so each repair reacquires the selected backend's active permit
|
||||||
|
while remaining inside its original admission lease. Repair never performs a
|
||||||
|
second bounded admission, and repaired outputs use the operation's existing
|
||||||
|
validation plan. The runner stops at the first valid candidate, sums completed
|
||||||
|
generation usage, reports calls actually started, and returns the final failed
|
||||||
|
validation result on exhaustion. A repair generation failure follows the
|
||||||
|
ordinary generation-error category rather than becoming a validation error.
|
||||||
|
|
||||||
A successful result includes the output artifact and raw output, validation
|
A successful result includes the output artifact and raw output, validation
|
||||||
state, effective session ID, prompt and rendered-prompt hashes, selected
|
state, effective session ID, prompt and rendered-prompt hashes, selected
|
||||||
|
|||||||
@@ -39,17 +39,17 @@ duplicate detection, and source containment:
|
|||||||
|
|
||||||
## Profiles And Built-Ins
|
## Profiles And Built-Ins
|
||||||
|
|
||||||
`internal/profile` loads and validates execution profiles from an
|
`internal/profile` loads, locally validates, overlays, and resolves execution
|
||||||
operating-system filesystem or an `fs.FS`. A file contains exactly one YAML
|
profiles from an operating-system filesystem or an `fs.FS`. A file contains
|
||||||
document and its trimmed YAML `id` is its only selection identity; filenames do
|
exactly one YAML document and its trimmed YAML `id` is its only selection
|
||||||
not confer authority. Each point lookup reads discovered files once for their
|
identity; filenames do not confer authority. Each point lookup reads discovered
|
||||||
metadata and reuses the selected file's bytes for strict decoding; unrelated
|
files once for their metadata and reuses the selected file's bytes for strict
|
||||||
profiles are not fully decoded. Strict selected decoding recognizes the
|
decoding; unrelated profiles are not fully decoded. Strict selected decoding
|
||||||
optional `backend` field, trims its value, and requires a model plus at least
|
recognizes `base_profile` and the optional `backend` field, trims their values,
|
||||||
one non-blank backend or endpoint. File-backed `extra_params` values are
|
and permits inherited target fields only when a base is named. File-backed
|
||||||
validated and defensively copied through the shared bounded JSON-value owner
|
`extra_params` values are validated and defensively copied through the shared
|
||||||
before a profile is published. OpenAI-compatible reserved-field policy remains
|
bounded JSON-value owner before a profile is published. OpenAI-compatible
|
||||||
with the model-client and backend-registry owners.
|
reserved-field policy remains with the model-client and backend-registry owners.
|
||||||
|
|
||||||
The overlay repository consults the next repository only when the
|
The overlay repository consults the next repository only when the
|
||||||
higher-precedence repository reports that a profile is absent. A reliably
|
higher-precedence repository reports that a profile is absent. A reliably
|
||||||
@@ -59,14 +59,25 @@ backend registry membership because the available registry belongs to the
|
|||||||
assembled engine; the runner checks membership during preparation and exact
|
assembled engine; the runner checks membership during preparation and exact
|
||||||
profile inspection.
|
profile inspection.
|
||||||
|
|
||||||
The root engine assembles profile repositories in precedence order: in-memory
|
The root engine assembles one raw composite catalog in precedence order:
|
||||||
profiles, one ordinary configured source, an application fallback source, then
|
in-memory profiles, one ordinary configured source, an application fallback
|
||||||
the embedded built-in catalog. An explicit file or `fs.FS` profile source
|
source, then the embedded built-in catalog. An explicit file or `fs.FS` profile
|
||||||
replaces `Config.ProfileDir` within the ordinary configured-source category.
|
source replaces `Config.ProfileDir` within the ordinary configured-source
|
||||||
|
category. One outer resolving repository wraps that complete raw catalog, so
|
||||||
|
each base lookup observes the same precedence and shadowing rules.
|
||||||
|
|
||||||
Exact profile inspection performs one point-in-time lookup through those
|
The resolving repository traverses every selected chain afresh, retains no
|
||||||
profile sources and checks the resolved target without reading prompt, input,
|
cache, detects cycles, limits a chain to 32 profiles, merges root-to-leaf into a
|
||||||
|
new caller-owned value, and validates the final target before publishing it. It
|
||||||
|
does not check backend registry membership. Exact `base_profile` syntax, merge
|
||||||
|
rules, and consumer-visible failure behavior belong to the [framework format
|
||||||
|
reference](../formats.md#profile-inheritance).
|
||||||
|
|
||||||
|
Exact profile inspection performs one point-in-time resolved lookup through
|
||||||
|
those profile sources and checks the final target without reading prompt, input,
|
||||||
or schema sources. It does not retain that lookup for a later execution.
|
or schema sources. It does not retain that lookup for a later execution.
|
||||||
|
Prepared execution instead freezes the fully resolved target; a later ordinary
|
||||||
|
operation performs a fresh traversal.
|
||||||
|
|
||||||
`internal/profile/builtin` embeds the maintained built-in profile catalog.
|
`internal/profile/builtin` embeds the maintained built-in profile catalog.
|
||||||
Every embedded profile selects a maintained built-in backend and inherits that
|
Every embedded profile selects a maintained built-in backend and inherits that
|
||||||
|
|||||||
155
docs/releases/v0.7.0.md
Normal file
155
docs/releases/v0.7.0.md
Normal file
@@ -0,0 +1,155 @@
|
|||||||
|
# Promptkit v0.7.0
|
||||||
|
|
||||||
|
This supplemental changelog and migration guide summarizes the consumer-facing
|
||||||
|
changes from `v0.6.0` to `v0.7.0`. The annotated `v0.7.0` tag is the
|
||||||
|
authoritative release record. Exact current contracts belong to the linked
|
||||||
|
GoDoc and durable documentation.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
`v0.7.0` expands provider integration and profile composition while making
|
||||||
|
credential and generation-failure handling more flexible:
|
||||||
|
|
||||||
|
- Promptkit now includes the `rakestrawhome` backend and its Gemma profile;
|
||||||
|
- built-in generation failures expose bounded structured provider details;
|
||||||
|
- an unavailable optional API-key environment source no longer prevents a
|
||||||
|
request from reaching an upstream that permits unauthenticated access; and
|
||||||
|
- profiles can inherit from and selectively refine another profile.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
This release adds public declarations and fields but removes none. Existing
|
||||||
|
keyed configuration literals and ordinary `errors.Is` handling continue to
|
||||||
|
work.
|
||||||
|
|
||||||
|
Adding `BaseProfileID` to `Profile` and `OpenAICompatibleProfileConfig` changes
|
||||||
|
their struct shape. Consumers using positional composite literals for either
|
||||||
|
type must convert them to keyed literals. Existing keyed literals require no
|
||||||
|
change.
|
||||||
|
|
||||||
|
The `rakestrawhome` backend ID is now built in and reserved. A consumer that
|
||||||
|
previously registered that exact ID with `WithBackend` must remove its manual
|
||||||
|
registration before upgrading. Other custom backend registrations are
|
||||||
|
unchanged.
|
||||||
|
|
||||||
|
When an optional backend, profile, or request `APIKeyEnv` is unset, empty, or
|
||||||
|
whitespace-only, the built-in client now omits `Authorization` and sends the
|
||||||
|
request. Previously this condition could fail before transport. Set
|
||||||
|
`Profile.APIKeyRequired` when missing credentials must remain a local
|
||||||
|
preflight error.
|
||||||
|
|
||||||
|
Provider non-success responses continue to match `ErrLLMGenerate`. Their
|
||||||
|
rendered wording is not a compatibility contract; consumers can now use
|
||||||
|
`errors.As` with `*GenerationError` when structured status information is
|
||||||
|
needed.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
Update the module dependency with:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get gitea.maximumdirect.net/eric/promptkit@v0.7.0
|
||||||
|
go mod tidy
|
||||||
|
```
|
||||||
|
|
||||||
|
Remove any manual `rakestrawhome` backend registration, convert positional
|
||||||
|
profile literals to keyed literals, and run the consuming project's ordinary
|
||||||
|
and race-enabled tests.
|
||||||
|
|
||||||
|
## Rakestrawhome Built-In Backend And Profile
|
||||||
|
|
||||||
|
Every engine now includes the reserved `rakestrawhome` backend, identified by
|
||||||
|
`BackendRakestrawHome`. The built-in `rakestrawhome-gemma-4-31b` profile
|
||||||
|
selects that backend. Consumers can use the maintained endpoint, credential,
|
||||||
|
capacity, and model defaults without registering either definition themselves.
|
||||||
|
|
||||||
|
See the [built-in backend and profile catalogs](../formats.md#built-in-backends)
|
||||||
|
and the [consumer adoption example](../consumers/pkg-promptkit.md#use-the-rakestrawhome-built-in-profile)
|
||||||
|
for the current contracts.
|
||||||
|
|
||||||
|
## Structured Generation Errors
|
||||||
|
|
||||||
|
Non-2xx responses from the built-in OpenAI-compatible client now return an
|
||||||
|
immutable `*GenerationError`. Consumers can inspect the HTTP status and any
|
||||||
|
safely extracted provider code, type, or message while retaining the ordinary
|
||||||
|
generation-error category:
|
||||||
|
|
||||||
|
```go
|
||||||
|
var generationErr *promptkit.GenerationError
|
||||||
|
if errors.As(err, &generationErr) {
|
||||||
|
status := generationErr.StatusCode()
|
||||||
|
_ = status
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Provider fields are bounded and normalized but remain untrusted and may
|
||||||
|
contain sensitive request or schema details. Default and Go-syntax formatting
|
||||||
|
omit those fields. Applications must apply their own disclosure policy before
|
||||||
|
logging or presenting accessor values.
|
||||||
|
|
||||||
|
See the [`GenerationError` GoDoc](../../generation_error.go), the
|
||||||
|
[consumer error-handling guide](../consumers/pkg-promptkit.md#handle-errors),
|
||||||
|
and the [OpenAI-compatible response contract](../integrations/openai-compatible-chat.md#response-handling).
|
||||||
|
|
||||||
|
## Optional Credential Sources
|
||||||
|
|
||||||
|
`APIKeyEnv` names an optional environment lookup source unless the selected
|
||||||
|
profile explicitly sets `APIKeyRequired`. When neither a direct request key nor
|
||||||
|
a usable environment value exists, the built-in client omits the bearer header
|
||||||
|
and handles the upstream response normally. This supports local and other
|
||||||
|
OpenAI-compatible providers that permit unauthenticated requests without
|
||||||
|
hiding an authentication error returned by a provider that requires one.
|
||||||
|
|
||||||
|
The [credential format reference](../formats.md#credentials), the
|
||||||
|
[`Backend` GoDoc](../../backends.go), the
|
||||||
|
[`ExecutionTargetOverride` GoDoc](../../types.go), and the
|
||||||
|
[authentication integration contract](../integrations/openai-compatible-chat.md#authentication)
|
||||||
|
define the current precedence and availability rules.
|
||||||
|
|
||||||
|
## Profile Inheritance
|
||||||
|
|
||||||
|
YAML profiles can name one parent with `base_profile`; in-memory profiles use
|
||||||
|
`Profile.BaseProfileID`, and `OpenAICompatibleProfileConfig` forwards the same
|
||||||
|
field. A profile can act as an application-owned alias of a built-in or refine
|
||||||
|
selected inherited settings:
|
||||||
|
|
||||||
|
```go
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "weather-light",
|
||||||
|
BaseProfileID: "deepseek-4-flash",
|
||||||
|
ReasoningEffort: "high",
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
Base lookup observes the existing source precedence. Chains are linear,
|
||||||
|
cycle-safe, and resolved afresh for ordinary operations. Prepared execution
|
||||||
|
freezes the fully resolved target. The selected leaf ID remains public while
|
||||||
|
effective execution settings reflect the resolved chain.
|
||||||
|
|
||||||
|
See the [profile inheritance format reference](../formats.md#profile-inheritance),
|
||||||
|
the [consumer alias example](../consumers/pkg-promptkit.md#alias-a-built-in-profile),
|
||||||
|
and the [`Profile` GoDoc](../../types.go) for exact merge and validation
|
||||||
|
behavior.
|
||||||
|
|
||||||
|
## Public API Changes
|
||||||
|
|
||||||
|
The release adds:
|
||||||
|
|
||||||
|
- `BackendRakestrawHome`;
|
||||||
|
- `GenerationError`, including `StatusCode`, `ProviderCode`, `ProviderType`,
|
||||||
|
`ProviderMessage`, `Error`, `GoString`, and `Unwrap`;
|
||||||
|
- `Profile.BaseProfileID`; and
|
||||||
|
- `OpenAICompatibleProfileConfig.BaseProfileID`.
|
||||||
|
|
||||||
|
No public declaration was removed.
|
||||||
|
|
||||||
|
## Consumer Action
|
||||||
|
|
||||||
|
- Remove a manual backend registration whose ID is exactly `rakestrawhome`.
|
||||||
|
- Convert positional `Profile` or `OpenAICompatibleProfileConfig` literals to
|
||||||
|
keyed literals.
|
||||||
|
- Set `Profile.APIKeyRequired` where a missing credential must fail locally
|
||||||
|
instead of reaching the provider unauthenticated.
|
||||||
|
- Treat `GenerationError` provider fields as untrusted and potentially
|
||||||
|
sensitive when adopting the new accessors.
|
||||||
|
- Run consumer ordinary and race-enabled tests after updating the module.
|
||||||
109
docs/releases/v0.8.0.md
Normal file
109
docs/releases/v0.8.0.md
Normal file
@@ -0,0 +1,109 @@
|
|||||||
|
# Promptkit v0.8.0
|
||||||
|
|
||||||
|
This supplemental changelog and migration guide summarizes the consumer-facing
|
||||||
|
changes from `v0.7.0` to `v0.8.0`. The annotated `v0.8.0` tag is the
|
||||||
|
authoritative release record. Exact current contracts belong to the linked
|
||||||
|
GoDoc and durable documentation.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
`v0.8.0` activates Promptkit's bounded output-repair workflow:
|
||||||
|
|
||||||
|
- failed nonempty-text, JSON, and JSON Schema validation can make a limited
|
||||||
|
number of corrective model calls;
|
||||||
|
- corrective calls preserve the original rendered conversation, effective
|
||||||
|
target, session, structured-output contract, and backend capacity policy;
|
||||||
|
- results report cumulative usage and the number of corrective calls actually
|
||||||
|
made; and
|
||||||
|
- explicitly empty OpenAI-compatible response content now reaches output
|
||||||
|
validation instead of being classified as a malformed provider envelope.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
This release adds no public declarations or fields and removes none. Existing
|
||||||
|
source code remains source-compatible.
|
||||||
|
|
||||||
|
The behavior of the existing `OutputContract.RepairAttempts` field and prompt
|
||||||
|
YAML `repair_attempts` field has changed. A positive value now authorizes real
|
||||||
|
additional model calls after eligible validation failures; earlier releases
|
||||||
|
accepted the field but the public engine remained single-pass. Consumers that
|
||||||
|
set a positive value should expect additional latency, token usage, and
|
||||||
|
provider cost when repair is needed.
|
||||||
|
|
||||||
|
Repair budgets must now be between zero and three. A positive budget requires
|
||||||
|
`basic`, `json`, or `json_schema` validation. Values above three and a positive
|
||||||
|
budget paired with `none` are invalid contracts rather than ignored settings.
|
||||||
|
|
||||||
|
An explicitly present empty or whitespace-only string returned by the built-in
|
||||||
|
OpenAI-compatible client is now a completed generation candidate. `none`
|
||||||
|
validation permits it, while `basic`, `json`, and `json_schema` classify it
|
||||||
|
under their ordinary validation rules and may repair it when configured.
|
||||||
|
Missing, `null`, or non-string content remains a malformed provider response.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
Update the module dependency with:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get gitea.maximumdirect.net/eric/promptkit@v0.8.0
|
||||||
|
go mod tidy
|
||||||
|
```
|
||||||
|
|
||||||
|
Review every prompt definition and request override that sets a positive repair
|
||||||
|
budget. Use zero or omit the field to retain single-pass execution. Ensure each
|
||||||
|
positive budget is no greater than three and uses an eligible validation mode,
|
||||||
|
then run the consuming project's ordinary and race-enabled tests.
|
||||||
|
|
||||||
|
## Bounded Output Repair
|
||||||
|
|
||||||
|
`repair_attempts` counts corrective calls in addition to the initial model
|
||||||
|
call. Promptkit validates each completed candidate, stops at the first valid
|
||||||
|
one, and never exceeds the configured bound. If every candidate remains
|
||||||
|
invalid, the run completes successfully with the final candidate and its
|
||||||
|
failed validation result rather than returning an operational error.
|
||||||
|
|
||||||
|
Each correction starts from the original rendered messages and includes only
|
||||||
|
the latest invalid candidate and latest validation diagnostics. JSON Schema
|
||||||
|
mode retains the provider-native structured-output request as its first line of
|
||||||
|
defense. Promptkit performs only deterministic structural validation; a valid
|
||||||
|
response is not necessarily factual or correct for an application's domain.
|
||||||
|
|
||||||
|
Usage in the final result is cumulative across the initial response and every
|
||||||
|
completed corrective response. `ValidationResult.RepairAttempts` reports the
|
||||||
|
number of corrective calls actually made. Corrective generation failures use
|
||||||
|
the same public generation-error categories and structured provider details as
|
||||||
|
an initial generation failure.
|
||||||
|
|
||||||
|
See the [output-contract format reference](../formats.md#output-contract), the
|
||||||
|
[consumer repair example](../consumers/pkg-promptkit.md#repair-a-structured-result),
|
||||||
|
and the [`OutputContract` and `ValidationResult` GoDoc](../../types.go) for the
|
||||||
|
current contracts.
|
||||||
|
|
||||||
|
## Explicit Empty Content
|
||||||
|
|
||||||
|
The built-in OpenAI-compatible client now distinguishes an explicitly present
|
||||||
|
empty string from a missing or malformed `content` field. This aligns built-in
|
||||||
|
and injected clients by letting the selected output contract decide whether an
|
||||||
|
empty candidate is acceptable, invalid, or eligible for repair.
|
||||||
|
|
||||||
|
See the
|
||||||
|
[OpenAI-compatible response contract](../integrations/openai-compatible-chat.md#response-handling)
|
||||||
|
for the exact envelope behavior.
|
||||||
|
|
||||||
|
## Public API Changes
|
||||||
|
|
||||||
|
None. This release activates and tightens the documented behavior of existing
|
||||||
|
fields.
|
||||||
|
|
||||||
|
## Consumer Action
|
||||||
|
|
||||||
|
- Remove or set `repair_attempts` to zero where execution must remain
|
||||||
|
single-pass.
|
||||||
|
- Keep every positive repair budget at three or fewer and pair it with
|
||||||
|
`basic`, `json`, or `json_schema` validation.
|
||||||
|
- Account for additional latency, usage, and provider cost when enabling
|
||||||
|
repair.
|
||||||
|
- Continue checking the returned validation status because bounded repair can
|
||||||
|
exhaust without producing a valid candidate.
|
||||||
|
- Review workflows that previously treated explicit empty provider content as
|
||||||
|
a generation error.
|
||||||
@@ -38,33 +38,8 @@ consumers.
|
|||||||
|
|
||||||
## Ideas
|
## Ideas
|
||||||
|
|
||||||
### Public bounded output repair
|
No ideas are currently awaiting selection. Active feature work belongs in its
|
||||||
|
focused roadmap rather than this catalog.
|
||||||
After the codebase-audit remediations are complete, Promptkit should make its
|
|
||||||
bounded output-repair capability available through the public engine. A
|
|
||||||
consumer should be able to request a limited number of corrective generation
|
|
||||||
attempts when JSON or JSON Schema output fails content validation, without
|
|
||||||
having to reproduce Promptkit's generation, validation, capacity, and result-
|
|
||||||
accounting orchestration.
|
|
||||||
|
|
||||||
- Repair is validation recovery, not a general provider retry, failover, or
|
|
||||||
backoff policy. Transport failures, cancellation, and operational schema or
|
|
||||||
validation errors must retain their ordinary error behavior.
|
|
||||||
- Repair must stop after the first valid result or the configured attempt
|
|
||||||
bound. Exhausting the bound should preserve the final invalid result and its
|
|
||||||
validation diagnostics rather than inventing success.
|
|
||||||
- Initial generation and every repair attempt must use the same resolved
|
|
||||||
backend, effective execution settings and presence semantics, session,
|
|
||||||
credential boundary, structured-output contract, and backend-capacity
|
|
||||||
policy.
|
|
||||||
- Results should report the number of repair attempts and cumulative usage for
|
|
||||||
every model call made by the run.
|
|
||||||
- Ordinary and prepared execution should expose coherent behavior, including
|
|
||||||
cancellation, frozen prepared state, error identity, and capacity lifetime.
|
|
||||||
|
|
||||||
Select this work only after the accepted audit findings affecting shared
|
|
||||||
execution invariants, validation, orchestration, transport, and repair
|
|
||||||
internals have been remediated.
|
|
||||||
|
|
||||||
## Entry Format
|
## Entry Format
|
||||||
|
|
||||||
|
|||||||
@@ -1,437 +0,0 @@
|
|||||||
# Structured Generation Errors Implementation Plan
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
Implement the target state defined in the
|
|
||||||
[structured generation errors roadmap](structured-generation-errors.md): turn
|
|
||||||
every non-2xx response from the built-in OpenAI-compatible client into a
|
|
||||||
bounded, immutable public error that exposes deliberate provider diagnostics
|
|
||||||
through `errors.As` while retaining `ErrLLMGenerate` through `errors.Is`.
|
|
||||||
|
|
||||||
This document owns implementation sequencing. The feature roadmap owns the
|
|
||||||
consumer intent, public-policy decisions, safety limits, compatibility
|
|
||||||
boundaries, and non-goals. Follow the architecture, documentation, and testing
|
|
||||||
policies under [`docs/policy/`](../policy/) throughout the work.
|
|
||||||
|
|
||||||
## Fixed Decisions
|
|
||||||
|
|
||||||
- The public type is `GenerationError`. Engine-produced values are pointers;
|
|
||||||
fields are unexported; no public constructor or mutation API is added.
|
|
||||||
- Public methods are `StatusCode() int`, `ProviderCode() string`,
|
|
||||||
`ProviderType() string`, `ProviderMessage() string`, `Error() string`,
|
|
||||||
`GoString() string`, and `Unwrap() error`.
|
|
||||||
- Public `Unwrap` returns `ErrLLMGenerate`. Accessors, formatting, and unwrapping
|
|
||||||
are safe on a nil receiver and a zero value.
|
|
||||||
- Engine-produced `Error()` text is exactly
|
|
||||||
`failed to generate output: provider returned HTTP status N`, where `N` is
|
|
||||||
the received status. A nil receiver or zero status returns exactly
|
|
||||||
`failed to generate output`. `GoString()` returns the same redacted text as
|
|
||||||
`Error()` so `%#v` cannot reveal unexported provider fields.
|
|
||||||
- Default formatting contains no provider-controlled code, type, message, or
|
|
||||||
raw response content. The type has no stable JSON representation.
|
|
||||||
- The recognized body is one JSON document with a top-level object-valued
|
|
||||||
`error`. `message` and `type` accept strings; `code` accepts a string or an
|
|
||||||
exact `json.Number`; supported fields are independent and unknown fields are
|
|
||||||
ignored.
|
|
||||||
- The non-success-body limit is 65,536 bytes. Declared oversize bodies are not
|
|
||||||
read; other bodies are read through a 65,537-byte bound. Oversize, malformed,
|
|
||||||
unrecognized, or unreadable bodies produce status-only detail.
|
|
||||||
- Normalization converts invalid UTF-8, collapses Unicode whitespace, control,
|
|
||||||
and format-character runs to one ASCII space, trims the result, omits blank
|
|
||||||
values, and produces one-line strings.
|
|
||||||
- Codes and types longer than 256 Unicode code points are omitted. Messages
|
|
||||||
longer than 4,096 code points retain the first 4,095 code points followed by
|
|
||||||
`…`, for a total limit of 4,096.
|
|
||||||
- Only the concrete built-in transport error is converted into a new public
|
|
||||||
`GenerationError`. Arbitrary injected-client errors are never inspected or
|
|
||||||
enriched.
|
|
||||||
- The use-case and domain layers remain provider-neutral. No retryability,
|
|
||||||
retry, logging, presentation, provider-specific envelope, header, or success-
|
|
||||||
response behavior is added.
|
|
||||||
|
|
||||||
## Execution Rules
|
|
||||||
|
|
||||||
- Complete the stages in numerical order. Each stage is scoped for one
|
|
||||||
gpt-5.6-terra implementation prompt and must finish its focused tests before
|
|
||||||
the next begins.
|
|
||||||
- Treat all stages as one feature delivery. Intermediate stages intentionally
|
|
||||||
create internal machinery before exposing it; do not release, tag, or claim
|
|
||||||
the feature is available until Stage 5 is complete.
|
|
||||||
- At the start of each stage, reread the feature roadmap and the task-specific
|
|
||||||
references in [`docs/development.md`](../development.md). Preserve unrelated
|
|
||||||
working-tree changes.
|
|
||||||
- Use classical behavior tests at the narrowest owner. Table-drive parser and
|
|
||||||
boundary cases, use controlled transports or local servers, and do not
|
|
||||||
duplicate the internal envelope matrix at the root engine boundary.
|
|
||||||
- Do not contact a live provider, add dependencies, commit, tag, push, or edit
|
|
||||||
release documentation unless separately instructed.
|
|
||||||
- Current-state prose documentation changes in Stage 5. Public GoDoc changes
|
|
||||||
alongside the public declarations in Stage 4 because GoDoc owns that API.
|
|
||||||
|
|
||||||
## Stage 1: Add the Internal Structured Status Error and Envelope Parser
|
|
||||||
|
|
||||||
### Objective
|
|
||||||
|
|
||||||
Create the transport-owned structured value and pure parsing and normalization
|
|
||||||
logic without changing `OpenAICompatibleClient.Generate` yet.
|
|
||||||
|
|
||||||
### Implementation
|
|
||||||
|
|
||||||
1. Add `internal/llm/provider_http_error.go`. Keep all provider HTTP mechanics
|
|
||||||
in `internal/llm`; do not add an HTTP error DTO to `internal/domain` or
|
|
||||||
`internal/usecase`.
|
|
||||||
2. Define these private constants:
|
|
||||||
- `maxProviderErrorResponseBytes int64 = 64 << 10`;
|
|
||||||
- `maxProviderErrorIdentifierRunes = 256`; and
|
|
||||||
- `maxProviderErrorMessageRunes = 4096`.
|
|
||||||
3. Add an exported-within-`internal` `ProviderHTTPError` type with unexported
|
|
||||||
`statusCode`, `providerCode`, `providerType`, and `providerMessage` fields.
|
|
||||||
The root facade will need to name this concrete type in Stage 4, but no
|
|
||||||
representation is public outside the module's `internal` boundary.
|
|
||||||
4. Give `ProviderHTTPError` nil-safe read-only accessors with the same four
|
|
||||||
names as the planned public type. Implement:
|
|
||||||
- `Error()` as `llm returned non-success status: status=N` when status is
|
|
||||||
nonzero and `llm returned non-success status` otherwise;
|
|
||||||
- `GoString()` by returning `Error()`; and
|
|
||||||
- `Unwrap()` by returning `ErrUnexpectedStatus`.
|
|
||||||
Never include provider-derived strings in either formatter.
|
|
||||||
5. Add a private `providerErrorDetails` value and a private constructor that
|
|
||||||
builds `*ProviderHTTPError` from a status plus already normalized details.
|
|
||||||
6. Add `parseProviderErrorEnvelope([]byte) providerErrorDetails` with these
|
|
||||||
rules:
|
|
||||||
- use `json.Decoder` with `UseNumber` and require EOF after trailing JSON
|
|
||||||
whitespace;
|
|
||||||
- require a top-level object and object-valued `error` member;
|
|
||||||
- retain supported fields as `json.RawMessage` so each can be decoded and
|
|
||||||
validated independently;
|
|
||||||
- accept string `message` and `type` values;
|
|
||||||
- accept string or `json.Number` `code`, preserving validated number text
|
|
||||||
without float conversion;
|
|
||||||
- ignore unknown fields and treat invalid supported-field values as absent;
|
|
||||||
and
|
|
||||||
- return empty details for malformed framing, a missing or invalid `error`
|
|
||||||
object, or an object with no usable fields.
|
|
||||||
7. Add private normalization helpers that implement the roadmap's UTF-8,
|
|
||||||
single-line, whitespace/control/format handling and exact rune limits. Use
|
|
||||||
rune-aware operations; do not truncate bytes in the middle of UTF-8. Omit
|
|
||||||
overlong code and type identifiers, and truncate overlong messages to 4,095
|
|
||||||
code points plus `…`.
|
|
||||||
|
|
||||||
### Tests
|
|
||||||
|
|
||||||
Add `internal/llm/provider_http_error_test.go` in package `llm` with focused,
|
|
||||||
table-driven tests:
|
|
||||||
|
|
||||||
1. `TestProviderHTTPErrorEnvelopeParsing` covers:
|
|
||||||
- all supported string fields;
|
|
||||||
- string, integer, fractional, and exponent-form numeric codes without
|
|
||||||
float coercion;
|
|
||||||
- `null` and invalid field types handled independently;
|
|
||||||
- unknown top-level and nested fields;
|
|
||||||
- missing, null, scalar, and empty `error` values;
|
|
||||||
- malformed, truncated, trailing-garbage, and second-document input; and
|
|
||||||
- no raw or unsupported metadata retained.
|
|
||||||
2. `TestProviderErrorTextNormalizationAndLimits` covers valid multibyte text,
|
|
||||||
invalid UTF-8 replacement, leading/trailing and repeated whitespace,
|
|
||||||
newline/tab/control/format characters, blank normalization, exact identifier
|
|
||||||
and message boundaries, identifier omission one rune over, and rune-safe
|
|
||||||
message ellipsis one rune over.
|
|
||||||
3. `TestProviderHTTPErrorIdentityAndFormatting` covers exact accessors,
|
|
||||||
`errors.Is(err, ErrUnexpectedStatus)`, nil receivers, zero values, and safe
|
|
||||||
`%v`, `%+v`, and `%#v` formatting with distinctive provider markers absent.
|
|
||||||
|
|
||||||
Do not test body reading, HTTP response ownership, the root public type, or
|
|
||||||
assembled engine behavior in this stage.
|
|
||||||
|
|
||||||
### Verification
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./internal/llm -run 'TestProvider'
|
|
||||||
go test ./internal/llm
|
|
||||||
go test ./...
|
|
||||||
```
|
|
||||||
|
|
||||||
Stage 1 is complete when the parser and internal error are fully protected but
|
|
||||||
the live non-2xx branch remains unchanged.
|
|
||||||
|
|
||||||
**Status:** Complete.
|
|
||||||
|
|
||||||
## Stage 2: Add the Bounded Non-Success Body Reader
|
|
||||||
|
|
||||||
### Objective
|
|
||||||
|
|
||||||
Implement and test bounded response-body extraction independently from HTTP
|
|
||||||
client integration, keeping status preservation separate from envelope
|
|
||||||
validity.
|
|
||||||
|
|
||||||
### Implementation
|
|
||||||
|
|
||||||
1. In `internal/llm/provider_http_error.go`, add a private helper with the
|
|
||||||
equivalent contract of:
|
|
||||||
|
|
||||||
```go
|
|
||||||
func providerHTTPErrorFromBody(
|
|
||||||
statusCode int,
|
|
||||||
contentLength int64,
|
|
||||||
body io.Reader,
|
|
||||||
) *ProviderHTTPError
|
|
||||||
```
|
|
||||||
|
|
||||||
2. Always return a nonnil `ProviderHTTPError` carrying `statusCode`.
|
|
||||||
3. When `contentLength` is greater than 65,536, return status-only detail
|
|
||||||
without reading `body`.
|
|
||||||
4. Otherwise read through an `io.LimitedReader` capped at 65,537 bytes. Return
|
|
||||||
status-only detail on a read error or when the extra byte is consumed. Do
|
|
||||||
not parse a bounded prefix of an incomplete oversized body.
|
|
||||||
5. For a complete body at or below the limit, call
|
|
||||||
`parseProviderErrorEnvelope` and construct the error from its normalized
|
|
||||||
details.
|
|
||||||
6. The helper does not close or drain `body`; the HTTP caller retains response-
|
|
||||||
body ownership. It must never read beyond the one-byte overflow probe.
|
|
||||||
|
|
||||||
### Tests
|
|
||||||
|
|
||||||
Extend `internal/llm/provider_http_error_test.go` with
|
|
||||||
`TestProviderHTTPErrorBodyBounds`, using counting, failing, and guarded readers
|
|
||||||
instead of an HTTP server. Cover:
|
|
||||||
|
|
||||||
- an ordinary recognized envelope;
|
|
||||||
- an exact 65,536-byte body, using trailing JSON whitespace to reach the
|
|
||||||
boundary while remaining one valid document;
|
|
||||||
- a declared 65,537-byte body with zero reads;
|
|
||||||
- unknown-length and underreported 65,537-byte bodies with exactly 65,537 bytes
|
|
||||||
read and status-only detail;
|
|
||||||
- an early read failure with status-only detail; and
|
|
||||||
- an empty body with status-only detail.
|
|
||||||
|
|
||||||
Assert the provider markers are absent whenever extraction is discarded. Do
|
|
||||||
not add body-closure assertions here because this helper does not own closing.
|
|
||||||
|
|
||||||
### Verification
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./internal/llm -run 'TestProviderHTTPErrorBodyBounds|TestProvider'
|
|
||||||
go test ./internal/llm
|
|
||||||
go test ./...
|
|
||||||
```
|
|
||||||
|
|
||||||
Stage 2 is complete when every read path is deterministically bounded and the
|
|
||||||
HTTP client's current branch is still untouched.
|
|
||||||
|
|
||||||
**Status:** Complete.
|
|
||||||
|
|
||||||
## Stage 3: Integrate Structured Status Errors into the Built-In Client
|
|
||||||
|
|
||||||
### Objective
|
|
||||||
|
|
||||||
Replace the built-in client's status-only discard branch with the bounded
|
|
||||||
internal error while preserving all successful, cancellation, transport, and
|
|
||||||
body-ownership behavior.
|
|
||||||
|
|
||||||
### Implementation
|
|
||||||
|
|
||||||
1. In `internal/llm/openai_compatible_client.go`, replace the non-2xx branch's
|
|
||||||
4,096-byte discard and formatted sentinel with
|
|
||||||
`providerHTTPErrorFromBody(httpResp.StatusCode, httpResp.ContentLength,
|
|
||||||
httpResp.Body)`.
|
|
||||||
2. Keep the existing `defer httpResp.Body.Close()` as the single body-closure
|
|
||||||
owner. Do not close in the helper, drain after the bound, or reuse the 16 MiB
|
|
||||||
successful-response decoder or limit.
|
|
||||||
3. Return no partial `GenerateResponse` for every non-2xx response.
|
|
||||||
4. Preserve `errors.Is(err, ErrUnexpectedStatus)` through
|
|
||||||
`ProviderHTTPError.Unwrap`. Do not change `requestFailedError`, endpoint or
|
|
||||||
request validation, authentication, timeout handling, successful response
|
|
||||||
decoding, or response-size behavior.
|
|
||||||
|
|
||||||
### Tests
|
|
||||||
|
|
||||||
1. Update the existing non-success case in
|
|
||||||
`internal/llm/openai_compatible_client_test.go` to assert
|
|
||||||
`errors.As(err, &providerHTTPError)`, exact status, and continued
|
|
||||||
`ErrUnexpectedStatus` identity. Keep the existing raw-body redaction check.
|
|
||||||
2. Add `internal/llm/provider_http_error_transport_test.go` with:
|
|
||||||
- `TestOpenAICompatibleClientStructuredNonSuccessResponse`, proving a
|
|
||||||
recognized envelope supplies all normalized internal fields and returns
|
|
||||||
no generation result;
|
|
||||||
- `TestOpenAICompatibleClientNonSuccessBodyOwnership`, table-driving normal,
|
|
||||||
declared-oversize, streamed-oversize, underreported, malformed, and read-
|
|
||||||
failure cases through a controlled transport; and
|
|
||||||
- assertions that every body is closed, no case reads beyond its bound,
|
|
||||||
declared oversize performs no read, and discarded details remain empty.
|
|
||||||
3. Reuse existing controlled-transport and counting-reader helpers when they
|
|
||||||
are clear and package-local. Do not duplicate successful-response framing,
|
|
||||||
timeout, authentication, or endpoint matrices.
|
|
||||||
|
|
||||||
### Verification
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./internal/llm -run 'TestOpenAICompatibleClient.*NonSuccess|TestProvider'
|
|
||||||
go test ./internal/llm
|
|
||||||
go test ./...
|
|
||||||
```
|
|
||||||
|
|
||||||
Stage 3 is complete when the built-in client emits the structured internal
|
|
||||||
error for every non-2xx response and all prior internal identities remain
|
|
||||||
green.
|
|
||||||
|
|
||||||
**Status:** Complete.
|
|
||||||
|
|
||||||
## Stage 4: Add the Public Error and Root Mapping Contract
|
|
||||||
|
|
||||||
### Objective
|
|
||||||
|
|
||||||
Translate only the built-in transport's structured error at the root facade,
|
|
||||||
publish the immutable consumer API, and prove ordinary, prepared, and injected-
|
|
||||||
client behavior.
|
|
||||||
|
|
||||||
### Implementation
|
|
||||||
|
|
||||||
1. Add `generation_error.go` in package `promptkit` with an immutable public
|
|
||||||
`GenerationError` whose four fields are unexported strings or integers. Add
|
|
||||||
one private constructor that accepts the four normalized scalar values. The
|
|
||||||
public error file must not import `internal/llm` or retain the internal error
|
|
||||||
or raw body; `errors.go` performs that adaptation at the facade boundary.
|
|
||||||
2. Implement the four public accessors exactly as fixed above. Each returns
|
|
||||||
zero or empty on a nil receiver.
|
|
||||||
3. Implement `Error()` with the fixed strings from this plan, `GoString()` by
|
|
||||||
returning `Error()`, and `Unwrap()` by returning `ErrLLMGenerate` even for a
|
|
||||||
nil receiver. Do not implement mutable fields, an exported constructor,
|
|
||||||
retry helpers, HTTP mapping, `fmt.Formatter`, or JSON methods.
|
|
||||||
4. Write complete GoDoc covering:
|
|
||||||
- built-in-client and non-2xx scope;
|
|
||||||
- ordinary and prepared execution;
|
|
||||||
- `errors.Is` and pointer-target `errors.As` usage;
|
|
||||||
- immutable and caller-owned semantics;
|
|
||||||
- nil and zero behavior;
|
|
||||||
- lack of stable JSON;
|
|
||||||
- safe default formatting; and
|
|
||||||
- the fact that every provider accessor is untrusted and may contain
|
|
||||||
sensitive request or schema fragments.
|
|
||||||
5. In `errors.go`, after the special capacity conversion and before generic
|
|
||||||
sentinel wrapping, use `errors.As` for a nonnil concrete
|
|
||||||
`*llm.ProviderHTTPError`. Convert it to `*GenerationError` and return that
|
|
||||||
public value directly. Do not parse error text or recognize an interface
|
|
||||||
that an injected client could accidentally satisfy.
|
|
||||||
6. Preserve the existing generic `publicErrorFor` path for all other failures.
|
|
||||||
In particular, arbitrary injected-client errors remain wrapped with
|
|
||||||
`ErrLLMGenerate` and retain their original identity.
|
|
||||||
7. Update public GoDoc in the same stage:
|
|
||||||
- the `ErrLLMGenerate` declaration points to `GenerationError` for built-in
|
|
||||||
non-2xx responses;
|
|
||||||
- `Engine.Run` and `Engine.RunPrepared` mention the typed error without
|
|
||||||
restating its accessors;
|
|
||||||
- `doc.go` distinguishes mutable `CapacityError` from immutable
|
|
||||||
`GenerationError`, lists both as lacking stable JSON, and calls out the
|
|
||||||
provider-detail trust boundary; and
|
|
||||||
- injected `LLMClient` GoDoc remains clear that arbitrary client errors are
|
|
||||||
preserved rather than translated.
|
|
||||||
|
|
||||||
### Tests
|
|
||||||
|
|
||||||
1. Add package-internal tests for `GenerationError` nil and zero receivers,
|
|
||||||
exact fixed formatting, and unwrapping. Do not expose a test-only public
|
|
||||||
constructor or turn the lack of stable JSON into a serialized-output
|
|
||||||
contract.
|
|
||||||
2. Add a focused external-package contract test, preferably in
|
|
||||||
`generation_error_contract_test.go`, that obtains errors through a real
|
|
||||||
assembled engine with a controlled HTTP transport:
|
|
||||||
- an ordinary `Run` with a recognized envelope asserts a nil result,
|
|
||||||
`errors.Is(err, ErrLLMGenerate)`, pointer-target `errors.As`, all four
|
|
||||||
accessors, exact status-bearing formatting, and absence of distinctive
|
|
||||||
code/type/message markers from `%v`, `%+v`, and `%#v`;
|
|
||||||
- one `RunPrepared` case proves the same public type and status cross the
|
|
||||||
prepared boundary without repeating every parser field; and
|
|
||||||
- neither case contacts a live provider or uses a real credential.
|
|
||||||
3. Extend the existing injected-client preservation owner with one assertion
|
|
||||||
that an arbitrary injected error does not become a `*GenerationError`, while
|
|
||||||
still matching both `ErrLLMGenerate` and the injected error.
|
|
||||||
4. Keep detailed envelope, normalization, size, and body-ownership matrices in
|
|
||||||
`internal/llm`; root tests remain representative.
|
|
||||||
|
|
||||||
### Verification
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test . -run 'TestGenerationError|TestBuiltInGenerationError|TestRunAddsLLMGenerate'
|
|
||||||
go test ./internal/llm
|
|
||||||
go test ./...
|
|
||||||
```
|
|
||||||
|
|
||||||
Stage 4 is complete when consumers can inspect built-in non-2xx details through
|
|
||||||
the stable root contract and injected errors remain untouched.
|
|
||||||
|
|
||||||
**Status:** Complete.
|
|
||||||
|
|
||||||
## Stage 5: Update Canonical Documentation and Run Full Validation
|
|
||||||
|
|
||||||
### Objective
|
|
||||||
|
|
||||||
Make durable documentation match the implemented contract, remove the roadmap
|
|
||||||
links that describe parsing as future work, and complete repository-wide
|
|
||||||
validation.
|
|
||||||
|
|
||||||
### Documentation
|
|
||||||
|
|
||||||
1. Update `docs/integrations/openai-compatible-chat.md` as the canonical wire
|
|
||||||
owner. Replace the status-only paragraph with:
|
|
||||||
- the recognized top-level envelope and independent field types;
|
|
||||||
- strict single-document framing and unknown-field behavior;
|
|
||||||
- the exact 65,536-byte read policy and status-only fallbacks;
|
|
||||||
- exact normalization and field limits;
|
|
||||||
- response closure and no-overread behavior; and
|
|
||||||
- the prohibition on raw bodies, headers, endpoints, credentials, request
|
|
||||||
data, schemas, and generated content.
|
|
||||||
2. Update `docs/internal/llm.md` with the internal `ProviderHTTPError`, bounded
|
|
||||||
reader and parser flow, retained `ErrUnexpectedStatus` identity, root
|
|
||||||
conversion boundary, and narrow test ownership. Remove its link that defers
|
|
||||||
parsing to the feature roadmap.
|
|
||||||
3. Update `docs/consumers/pkg-promptkit.md` under `Handle Errors` with one short
|
|
||||||
`errors.As` example. Show status and deliberate message access, warn that all
|
|
||||||
provider fields are untrusted and potentially sensitive, leave retry and
|
|
||||||
presentation policy to the application, and link to `GenerationError`
|
|
||||||
GoDoc rather than duplicating its full contract.
|
|
||||||
4. Update `docs/internal/overview.md` only enough to inventory implemented
|
|
||||||
responsibilities: the root facade owns typed capacity and generation error
|
|
||||||
mapping, and `internal/llm` owns bounded structured non-success response
|
|
||||||
decoding. Do not duplicate limits or accessor details there.
|
|
||||||
5. Do not change `docs/policy/architecture.md`, `docs/formats.md`, backend
|
|
||||||
documentation, release notes, or the README unless implementation reveals a
|
|
||||||
concrete inaccurate statement. Their canonical topics do not own this
|
|
||||||
contract.
|
|
||||||
|
|
||||||
### Final Validation
|
|
||||||
|
|
||||||
Run the complete maintainer workflow from
|
|
||||||
[`docs/development.md#maintainer-validation`](../development.md#maintainer-validation):
|
|
||||||
|
|
||||||
```sh
|
|
||||||
go test ./...
|
|
||||||
go test -race ./...
|
|
||||||
go vet ./...
|
|
||||||
go build ./...
|
|
||||||
go run ./examples/go-library/prepare
|
|
||||||
go run ./examples/go-library/run
|
|
||||||
```
|
|
||||||
|
|
||||||
Also perform the documented Go-formatting, local Markdown-link, repository-
|
|
||||||
hygiene, ignored-file, and credential scans. Review both example outputs and
|
|
||||||
confirm that all commands remain deterministic, offline, and credential-free.
|
|
||||||
|
|
||||||
Finally review the complete diff against the feature roadmap and confirm:
|
|
||||||
|
|
||||||
- every built-in non-2xx response produces a status-bearing public type;
|
|
||||||
- malformed and oversized bodies cannot erase status or leak partial content;
|
|
||||||
- provider-derived strings appear only through deliberate accessors;
|
|
||||||
- default and Go-syntax formatting are redacted;
|
|
||||||
- successful, cancellation, capacity, validation, repair, and injected-client
|
|
||||||
behavior is unchanged;
|
|
||||||
- no provider policy entered the use-case or domain layers; and
|
|
||||||
- each exact contract has one canonical documentation and test owner.
|
|
||||||
|
|
||||||
**Status:** Complete.
|
|
||||||
|
|
||||||
## Open Questions
|
|
||||||
|
|
||||||
None. The roadmap and this plan fix the public API, internal representation,
|
|
||||||
wire envelope, normalization and bounds, status fallback, formatting,
|
|
||||||
propagation, compatibility, documentation, and verification decisions required
|
|
||||||
for implementation.
|
|
||||||
@@ -1,260 +0,0 @@
|
|||||||
# Structured Generation Errors
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
Promptkit should give downstream applications actionable, machine-readable
|
|
||||||
details when the built-in OpenAI-compatible client receives a non-success HTTP
|
|
||||||
response. Today the client reports only the status code and discards the
|
|
||||||
provider response body. This makes ordinary configuration failures, such as an
|
|
||||||
unsupported strict JSON Schema keyword, unnecessarily difficult to diagnose.
|
|
||||||
|
|
||||||
This feature supplies bounded facts about the provider response. It does not
|
|
||||||
make retry, presentation, or logging decisions for consumers.
|
|
||||||
|
|
||||||
## Target End State
|
|
||||||
|
|
||||||
Every non-2xx response received by Promptkit's built-in OpenAI-compatible
|
|
||||||
client becomes a public typed generation error. A consumer can use
|
|
||||||
`errors.As` to obtain the HTTP status and any safely extracted provider fields,
|
|
||||||
and `errors.Is` continues to match `ErrLLMGenerate`.
|
|
||||||
|
|
||||||
The typed contract is available from both `Run` and `RunPrepared`. It is not
|
|
||||||
produced during preparation, which performs no model request. Successful
|
|
||||||
responses, transport failures before a response is received, cancellation,
|
|
||||||
capacity failures, validation failures, and nil responses from injected model
|
|
||||||
clients retain their existing categories and behavior.
|
|
||||||
|
|
||||||
An unusable response body never hides the known HTTP status. Empty, malformed,
|
|
||||||
unrecognized, unreadable, or oversized bodies therefore produce the same typed
|
|
||||||
error with status-only detail rather than falling back to an unstructured
|
|
||||||
error or becoming a malformed-success response.
|
|
||||||
|
|
||||||
## Public Contract
|
|
||||||
|
|
||||||
The root package exposes an immutable `GenerationError` type with unexported
|
|
||||||
state and these read-only accessors:
|
|
||||||
|
|
||||||
- `StatusCode() int` returns the received HTTP status code;
|
|
||||||
- `ProviderCode() string` returns a normalized provider code, when present;
|
|
||||||
- `ProviderType() string` returns a normalized provider error type, when
|
|
||||||
present; and
|
|
||||||
- `ProviderMessage() string` returns the bounded normalized diagnostic message,
|
|
||||||
when present.
|
|
||||||
|
|
||||||
The engine returns a `*GenerationError`, so the idiomatic inspection form is:
|
|
||||||
|
|
||||||
```go
|
|
||||||
var generationErr *promptkit.GenerationError
|
|
||||||
if errors.As(err, &generationErr) {
|
|
||||||
status := generationErr.StatusCode()
|
|
||||||
message := generationErr.ProviderMessage()
|
|
||||||
_, _ = status, message
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
There is no public constructor or mutation API. The type implements `error`,
|
|
||||||
unwraps to `ErrLLMGenerate`, and provides safe ordinary and Go-syntax
|
|
||||||
formatting. `Error()` and `GoString()` include the HTTP status but no provider-
|
|
||||||
controlled code, type, or message. Consumers must use the accessors
|
|
||||||
deliberately when they want provider details and must not classify failures by
|
|
||||||
matching error text.
|
|
||||||
|
|
||||||
The zero value and a nil `*GenerationError` receiver are safe: accessors return
|
|
||||||
zero or empty values, formatting returns a generic redacted generation-failure
|
|
||||||
description, and unwrapping still identifies `ErrLLMGenerate`. Engine-produced
|
|
||||||
values always have the non-2xx status received from the provider. The type has
|
|
||||||
no stable JSON representation.
|
|
||||||
|
|
||||||
All provider-derived strings remain untrusted even after normalization. GoDoc
|
|
||||||
must warn consumers that provider fields can contain sensitive request or
|
|
||||||
schema fragments and must not be logged, displayed, or returned to another
|
|
||||||
caller without an application-appropriate disclosure policy.
|
|
||||||
|
|
||||||
## Recognized Provider Envelope
|
|
||||||
|
|
||||||
Promptkit recognizes only the conventional OpenAI-compatible top-level error
|
|
||||||
object:
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"error": {
|
|
||||||
"message": "diagnostic text",
|
|
||||||
"type": "invalid_request_error",
|
|
||||||
"code": "unsupported_parameter"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
The envelope must be one JSON document followed only by JSON whitespace. The
|
|
||||||
top-level `error` value must be an object. Unknown top-level and error-object
|
|
||||||
fields are ignored. The optional supported fields are interpreted
|
|
||||||
independently:
|
|
||||||
|
|
||||||
- `message` and `type` must be JSON strings;
|
|
||||||
- `code` may be a JSON string or number and is exposed as normalized text;
|
|
||||||
numeric codes retain their validated JSON number text without floating-point
|
|
||||||
coercion; and
|
|
||||||
- `null`, booleans, arrays, objects, or otherwise invalid values are treated as
|
|
||||||
absent for that field.
|
|
||||||
|
|
||||||
An invalid optional field does not discard other valid supported fields. An
|
|
||||||
absent `error` object, malformed or multiply framed JSON, or an object with no
|
|
||||||
usable supported fields simply leaves all provider accessors empty while
|
|
||||||
preserving the typed status error.
|
|
||||||
|
|
||||||
Promptkit does not expose `param`, metadata objects, nested causes, headers, or
|
|
||||||
provider-specific extensions in this feature.
|
|
||||||
|
|
||||||
## Bounded Reading And Normalization
|
|
||||||
|
|
||||||
Non-success bodies have a separate fixed limit of 64 KiB (65,536 bytes). This
|
|
||||||
is intentionally much smaller than the successful completion-body limit while
|
|
||||||
remaining large enough for useful schema diagnostics.
|
|
||||||
|
|
||||||
- A declared `Content-Length` above the limit is rejected without reading the
|
|
||||||
body for detail extraction.
|
|
||||||
- Otherwise Promptkit reads at most one byte beyond the limit so streamed,
|
|
||||||
chunked, and underreported bodies are bounded.
|
|
||||||
- A body over the limit contributes no provider fields; Promptkit does not
|
|
||||||
parse or retain a prefix as though it were a complete envelope.
|
|
||||||
- Read failures likewise discard provider fields while preserving the status.
|
|
||||||
- The response body is closed on every outcome and is not drained beyond the
|
|
||||||
bounded read.
|
|
||||||
|
|
||||||
Extracted strings are converted to valid UTF-8, trimmed, and made single-line:
|
|
||||||
invalid UTF-8 is replaced, and runs of Unicode whitespace, control characters,
|
|
||||||
and formatting controls are replaced with one ASCII space. Empty normalized
|
|
||||||
values are treated as absent.
|
|
||||||
|
|
||||||
Normalized provider codes and types are retained only when they contain at
|
|
||||||
most 256 Unicode code points. Longer values are omitted rather than truncated
|
|
||||||
so consumers never classify on a fabricated partial identifier. A provider
|
|
||||||
message is limited to 4,096 Unicode code points; a longer normalized message is
|
|
||||||
truncated at a code-point boundary with a visible ellipsis inside that limit.
|
|
||||||
The raw response body and pre-normalized strings are never exposed or retained
|
|
||||||
in the public error.
|
|
||||||
|
|
||||||
## Error Propagation And Compatibility
|
|
||||||
|
|
||||||
- Every built-in-client non-2xx response matches `ErrLLMGenerate` and supports
|
|
||||||
`errors.As` to `*GenerationError`, including status-only cases.
|
|
||||||
- The internal model client retains its non-success-status identity for its
|
|
||||||
own package tests. The use-case layer remains provider-neutral and continues
|
|
||||||
to add only its generation category.
|
|
||||||
- The root error boundary converts only the built-in transport's structured
|
|
||||||
status error. It does not parse arbitrary error text, inspect consumer error
|
|
||||||
fields, or fabricate HTTP details for an injected `LLMClient`.
|
|
||||||
- Errors returned by injected clients remain in the chain exactly as today.
|
|
||||||
If an injected client deliberately returns an existing `*GenerationError`,
|
|
||||||
its identity may pass through ordinary wrapping, but Promptkit does not
|
|
||||||
construct or enrich one on that client's behalf.
|
|
||||||
- Existing cancellation and deadline identities, capacity errors, validation
|
|
||||||
behavior, repair behavior, and successful response decoding remain
|
|
||||||
unchanged.
|
|
||||||
- This is an additive public API. Existing consumers that use
|
|
||||||
`errors.Is(err, ErrLLMGenerate)` continue to work; consumers should not rely
|
|
||||||
on the previous rendered wording of non-success errors.
|
|
||||||
|
|
||||||
## Architecture And Ownership
|
|
||||||
|
|
||||||
The provider-envelope parser and bounded body reader belong in `internal/llm`,
|
|
||||||
which owns the OpenAI-compatible transport. The internal transport error owns
|
|
||||||
only normalized status facts and continues to match the package's existing
|
|
||||||
non-success-status sentinel.
|
|
||||||
|
|
||||||
The use-case package does not gain HTTP DTOs, status policy, or a provider-
|
|
||||||
specific branch. Its existing wrapping carries the internal error to the root
|
|
||||||
facade. The root error mapper recognizes the internal structured status error
|
|
||||||
and constructs the public `GenerationError` without exposing an internal type
|
|
||||||
or raw cause through public fields. No transport error is added to
|
|
||||||
`internal/domain`.
|
|
||||||
|
|
||||||
The public type and its exact Go semantics are owned by its declaration and
|
|
||||||
GoDoc. The
|
|
||||||
[OpenAI-compatible integration contract](../integrations/openai-compatible-chat.md)
|
|
||||||
owns recognized wire shapes, limits, and observable response behavior. The
|
|
||||||
[internal model-client document](../internal/llm.md) owns implementation flow,
|
|
||||||
internal failure categories, and test ownership. Architecture policy does not
|
|
||||||
need a new package or dependency rule for this feature.
|
|
||||||
|
|
||||||
## Documentation End State
|
|
||||||
|
|
||||||
Canonical documentation at the target state has these responsibilities:
|
|
||||||
|
|
||||||
- the `GenerationError` declaration and GoDoc define the exact public methods,
|
|
||||||
formatting, unwrapping, zero-value behavior, and trust boundary;
|
|
||||||
- `Engine.Run` and `Engine.RunPrepared` GoDoc identify the typed error without
|
|
||||||
duplicating its accessor contract;
|
|
||||||
- the consumer guide includes one short `errors.As` example and links to the
|
|
||||||
public declaration;
|
|
||||||
- the integration document replaces its status-only description with the
|
|
||||||
bounded envelope contract; and
|
|
||||||
- the internal model-client document describes parsing, conversion ownership,
|
|
||||||
and narrow test owners.
|
|
||||||
|
|
||||||
The architecture policy, framework format reference, and built-in backend
|
|
||||||
catalog do not duplicate this API or wire contract.
|
|
||||||
|
|
||||||
## Verification Expectations
|
|
||||||
|
|
||||||
Verification protects each behavior at its narrowest stable owner:
|
|
||||||
|
|
||||||
- internal model-client tests cover recognized string and numeric codes,
|
|
||||||
independent optional-field handling, unknown fields, empty and malformed
|
|
||||||
envelopes, single-document framing, read failures, declared and streamed
|
|
||||||
size boundaries, body closure, normalization, field limits, and absence of
|
|
||||||
raw provider content from rendered errors;
|
|
||||||
- root error-boundary tests cover conversion to the immutable public type,
|
|
||||||
every accessor, `errors.Is`, `errors.As`, and safe `%v`, `%+v`, and `%#v`
|
|
||||||
formatting;
|
|
||||||
- one representative ordinary run and one prepared run prove that the built-in
|
|
||||||
transport contract crosses the assembled engine boundary, without repeating
|
|
||||||
the complete parser matrix;
|
|
||||||
- existing injected-client tests continue to prove preservation of consumer
|
|
||||||
error identity without fabricated provider details; and
|
|
||||||
- all tests use controlled transports or local servers and never contact a
|
|
||||||
live or paid provider.
|
|
||||||
|
|
||||||
Security limits and their exact boundaries are contractual enough to warrant
|
|
||||||
literal boundary tests. Higher-level tests should remain representative and
|
|
||||||
must not duplicate the internal transport matrix.
|
|
||||||
|
|
||||||
## Acceptance Criteria
|
|
||||||
|
|
||||||
- A consumer can distinguish an HTTP 400 from other generation failures and
|
|
||||||
deliberately obtain a bounded provider explanation when one is available.
|
|
||||||
- The same typed error remains available through ordinary and prepared
|
|
||||||
execution and still satisfies `errors.Is(err, ErrLLMGenerate)`.
|
|
||||||
- Default and Go-syntax error formatting cannot disclose any provider-derived
|
|
||||||
string or raw response content.
|
|
||||||
- Empty, malformed, unreadable, unrecognized, and oversized bodies preserve a
|
|
||||||
typed status-only error.
|
|
||||||
- No read, retained field, or formatted representation can exceed its stated
|
|
||||||
bound, and the body is closed on every outcome.
|
|
||||||
- Existing success, cancellation, capacity, validation, repair, and injected-
|
|
||||||
client contracts remain unchanged.
|
|
||||||
- Current-state documentation changes only when the implementation exists and
|
|
||||||
follows the repository's canonical ownership policy.
|
|
||||||
|
|
||||||
## Non-Goals
|
|
||||||
|
|
||||||
This feature does not add:
|
|
||||||
|
|
||||||
- retryability classification, retry loops, backoff, failover, or routing;
|
|
||||||
- parsing of success bodies as errors or changes to successful-response limits;
|
|
||||||
- provider-specific envelope variants beyond the conventional top-level
|
|
||||||
`error` object;
|
|
||||||
- response headers such as `Retry-After`, raw bodies, request data, endpoints,
|
|
||||||
credentials, schema documents, generated content, or provider metadata;
|
|
||||||
- logging, telemetry, redaction policy for downstream applications, HTTP status
|
|
||||||
mapping for consumer servers, or user-facing presentation;
|
|
||||||
- translation or enrichment of arbitrary injected-client errors; or
|
|
||||||
- a new public package, public constructor, mutable error value, or transport
|
|
||||||
type in the domain model.
|
|
||||||
|
|
||||||
## Open Questions
|
|
||||||
|
|
||||||
None. The public type direction, accessor surface, formatting and error-chain
|
|
||||||
behavior, envelope scope, normalization, safety limits, fallback behavior,
|
|
||||||
layer ownership, compatibility boundaries, documentation ownership, and test
|
|
||||||
boundaries are fixed by this roadmap.
|
|
||||||
41
engine.go
41
engine.go
@@ -53,9 +53,9 @@ var (
|
|||||||
// an execution profile or resolve its backend, except for the profile
|
// an execution profile or resolve its backend, except for the profile
|
||||||
// not-found case represented by ErrProfileNotFound.
|
// not-found case represented by ErrProfileNotFound.
|
||||||
ErrProfileLoad = errors.New("failed to load execution profile")
|
ErrProfileLoad = errors.New("failed to load execution profile")
|
||||||
// ErrAPIKeyEnvMissing identifies an APIKeyEnv whose environment variable is
|
// ErrAPIKeyEnvMissing identifies an explicitly required APIKeyEnv whose
|
||||||
// unset or empty when no direct RunRequest.APIKey takes precedence. Such an
|
// environment variable is unset or empty after direct RunRequest.APIKey
|
||||||
// error also matches ErrInvalidRequest.
|
// precedence is applied. Such an error also matches ErrInvalidRequest.
|
||||||
ErrAPIKeyEnvMissing = errors.New("api_key_env points to an unset environment variable")
|
ErrAPIKeyEnvMissing = errors.New("api_key_env points to an unset environment variable")
|
||||||
// ErrArtifactLoad identifies a failure to resolve an input artifact. Errors
|
// ErrArtifactLoad identifies a failure to resolve an input artifact. Errors
|
||||||
// returned by an injected ArtifactReader remain available through errors.Is.
|
// returned by an injected ArtifactReader remain available through errors.Is.
|
||||||
@@ -304,10 +304,12 @@ func WithFallbackProfileFS(fsys fs.FS, root string) Option {
|
|||||||
// WithProfiles configures in-memory profiles that take precedence over
|
// WithProfiles configures in-memory profiles that take precedence over
|
||||||
// ordinary configured, application fallback, and built-in profiles.
|
// ordinary configured, application fallback, and built-in profiles.
|
||||||
//
|
//
|
||||||
// NewEngine validates and copies every profile. IDs must be unique within one
|
// NewEngine locally validates and copies every profile. IDs must be unique
|
||||||
// call. An invalid profile, duplicate ID, or unsupported ExtraParams value
|
// within one call. An invalid local definition, duplicate ID, or unsupported
|
||||||
// makes construction fail with ErrInvalidConfig. Repeating WithProfiles
|
// ExtraParams value makes construction fail with ErrInvalidConfig. A derived
|
||||||
// replaces the complete earlier in-memory set rather than merging it.
|
// profile's base reference and resolved target completeness are checked when it
|
||||||
|
// is selected or inspected. Repeating WithProfiles replaces the complete
|
||||||
|
// earlier in-memory set rather than merging it.
|
||||||
func WithProfiles(profiles ...Profile) Option {
|
func WithProfiles(profiles ...Profile) Option {
|
||||||
return optionFunc(func(options *engineOptions) error {
|
return optionFunc(func(options *engineOptions) error {
|
||||||
repo, err := newMemoryProfileRepository(profiles)
|
repo, err := newMemoryProfileRepository(profiles)
|
||||||
@@ -428,7 +430,7 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
return &Engine{
|
return &Engine{
|
||||||
runner: usecase.NewRunner(
|
runner: usecase.NewRunnerWithRepairer(
|
||||||
promptDefs,
|
promptDefs,
|
||||||
profiles,
|
profiles,
|
||||||
backendRegistry,
|
backendRegistry,
|
||||||
@@ -436,6 +438,7 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
|||||||
prompt.NewGoRenderer(),
|
prompt.NewGoRenderer(),
|
||||||
llmClient,
|
llmClient,
|
||||||
validator,
|
validator,
|
||||||
|
usecase.NewDefaultOutputRepairer(llmClient),
|
||||||
capacityManager,
|
capacityManager,
|
||||||
),
|
),
|
||||||
}, nil
|
}, nil
|
||||||
@@ -458,7 +461,7 @@ func newProfileRepository(profileDir string, options engineOptions) profile.Repo
|
|||||||
repository = profile.NewOverlayRepository(options.memoryProfiles, repository)
|
repository = profile.NewOverlayRepository(options.memoryProfiles, repository)
|
||||||
}
|
}
|
||||||
|
|
||||||
return repository
|
return profile.NewResolvingRepository(repository)
|
||||||
}
|
}
|
||||||
|
|
||||||
func fileSource(name string) (fs.FS, string, error) {
|
func fileSource(name string) (fs.FS, string, error) {
|
||||||
@@ -637,10 +640,12 @@ func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*Prepare
|
|||||||
// generated output.
|
// generated output.
|
||||||
//
|
//
|
||||||
// A content-validation failure is a successful run whose
|
// A content-validation failure is a successful run whose
|
||||||
// RunResult.Validation has Status ValidationFailed. An inability to perform
|
// RunResult.Validation has Status ValidationFailed. When its output contract
|
||||||
// validation returns an error matching ErrValidation and no partial result.
|
// has a positive repair budget, a failed eligible validation can make bounded
|
||||||
// The public Engine does not perform output repair, so validation is
|
// additional model calls and stops at the first valid candidate. Exhaustion
|
||||||
// single-pass even when OutputContract.RepairAttempts is positive.
|
// returns the final failed validation result with cumulative usage and actual
|
||||||
|
// repair attempts. An inability to generate or validate returns an error and
|
||||||
|
// no partial result.
|
||||||
//
|
//
|
||||||
// Run can return every error category documented by [Engine.Prepare], plus
|
// Run can return every error category documented by [Engine.Prepare], plus
|
||||||
// ErrCapacityExceeded and ErrLLMGenerate. An engine admission rejection is
|
// ErrCapacityExceeded and ErrLLMGenerate. An engine admission rejection is
|
||||||
@@ -684,17 +689,17 @@ func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
|||||||
//
|
//
|
||||||
// The supplied context governs this execution attempt independently of the
|
// The supplied context governs this execution attempt independently of the
|
||||||
// preparation context. It covers credential revalidation, admission,
|
// preparation context. It covers credential revalidation, admission,
|
||||||
// generation, validation, and any internal repair. Result timing begins after
|
// generation, validation, and any bounded output repair. Result timing begins
|
||||||
// the claim and excludes preparation and consumer-held delay.
|
// after the claim and excludes preparation and consumer-held delay.
|
||||||
//
|
//
|
||||||
// RunPrepared can return ErrInvalidRequest, ErrAPIKeyEnvMissing,
|
// RunPrepared can return ErrInvalidRequest, ErrAPIKeyEnvMissing,
|
||||||
// ErrCapacityExceeded, ErrLLMGenerate, or ErrValidation as applicable while
|
// ErrCapacityExceeded, ErrLLMGenerate, or ErrValidation as applicable while
|
||||||
// preserving documented collaborator and context identities. An engine
|
// preserving documented collaborator and context identities. An engine
|
||||||
// admission rejection is discoverable as [CapacityError] and still matches
|
// admission rejection is discoverable as [CapacityError] and still matches
|
||||||
// ErrCapacityExceeded. A built-in OpenAI-compatible non-2xx response is
|
// ErrCapacityExceeded. A built-in OpenAI-compatible non-2xx response is
|
||||||
// discoverable as [GenerationError]. A completed content-validation rejection
|
// discoverable as [GenerationError]. A completed content-validation rejection,
|
||||||
// is returned in RunResult, not as an operational error. An operational error
|
// including repair exhaustion, is returned in RunResult, not as an operational
|
||||||
// returns no partial RunResult.
|
// error. An operational error returns no partial RunResult.
|
||||||
func (e *Engine) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*RunResult, error) {
|
func (e *Engine) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*RunResult, error) {
|
||||||
if e == nil || e.runner == nil {
|
if e == nil || e.runner == nil {
|
||||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
|
|||||||
102
engine_test.go
102
engine_test.go
@@ -977,37 +977,72 @@ func TestPrepareDirectAPIKeyBypassesMissingEnvWithoutLeakingOrHashing(t *testing
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMissingCredentialsFailClearlyWhenProfileRequiresAuth(t *testing.T) {
|
func TestOptionalMissingCredentialsReachUpstream(t *testing.T) {
|
||||||
const missingEnv = "PROMPTKIT_PUBLIC_AUTH_MISSING"
|
const missingEnv = "PROMPTKIT_PUBLIC_AUTH_MISSING"
|
||||||
|
const providerBody = `{"error":{"message":"authentication failed","type":"authentication_error","code":"invalid_api_key"}}`
|
||||||
t.Setenv(missingEnv, "")
|
t.Setenv(missingEnv, "")
|
||||||
|
|
||||||
profileDir := t.TempDir()
|
called := false
|
||||||
writePublicProfileFileWithAPIKeyEnv(t, profileDir, "requires-auth", "http://localhost:8000/v1", "test-model", missingEnv)
|
config := promptkit.Config{
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
PromptDir: frameworkPromptDir,
|
||||||
PromptDir: frameworkPromptDir,
|
SchemaDir: frameworkSchemaDir,
|
||||||
ProfileDir: profileDir,
|
HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
|
||||||
SchemaDir: frameworkSchemaDir,
|
called = true
|
||||||
})
|
if values := req.Header.Values("Authorization"); len(values) != 0 {
|
||||||
|
t.Fatalf("Authorization values = %q, want absent", values)
|
||||||
|
}
|
||||||
|
return &http.Response{
|
||||||
|
StatusCode: http.StatusUnauthorized,
|
||||||
|
ContentLength: int64(len(providerBody)),
|
||||||
|
Body: io.NopCloser(strings.NewReader(providerBody)),
|
||||||
|
}, nil
|
||||||
|
})},
|
||||||
|
}
|
||||||
|
engine, err := promptkit.NewEngine(config,
|
||||||
|
promptkit.WithBackend(promptkit.Backend{
|
||||||
|
ID: "optional-auth",
|
||||||
|
Endpoint: "http://provider.test/v1",
|
||||||
|
APIKeyEnv: missingEnv,
|
||||||
|
}),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "optional-auth-profile",
|
||||||
|
BackendID: "optional-auth",
|
||||||
|
Model: "test-model",
|
||||||
|
}),
|
||||||
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected engine construction to succeed, got %v", err)
|
t.Fatalf("expected engine construction to succeed, got %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
_, err = engine.Prepare(context.Background(), promptkit.RunRequest{
|
result, err := engine.Run(context.Background(), promptkit.RunRequest{
|
||||||
PromptID: frameworkMarkdownSummaryPromptID,
|
PromptID: frameworkMarkdownSummaryPromptID,
|
||||||
ProfileID: "requires-auth",
|
ProfileID: "optional-auth-profile",
|
||||||
Inputs: map[string]promptkit.ArtifactRef{
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
"transcript": promptkit.Inline("Rin opens the gate."),
|
"transcript": promptkit.Inline("Rin opens the gate."),
|
||||||
"glossary": promptkit.Inline("gate: A guarded passage."),
|
"glossary": promptkit.Inline("gate: A guarded passage."),
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if !errors.Is(err, promptkit.ErrInvalidRequest) {
|
if !called {
|
||||||
t.Fatalf("expected invalid request for missing credentials, got %v", err)
|
t.Fatal("optional missing credential did not reach upstream")
|
||||||
}
|
}
|
||||||
if !errors.Is(err, promptkit.ErrAPIKeyEnvMissing) {
|
if result != nil {
|
||||||
t.Fatalf("expected missing credential environment error, got %v", err)
|
t.Fatalf("result = %+v, want nil", result)
|
||||||
}
|
}
|
||||||
if err == nil || !strings.Contains(err.Error(), missingEnv) {
|
if errors.Is(err, promptkit.ErrInvalidRequest) || errors.Is(err, promptkit.ErrAPIKeyEnvMissing) {
|
||||||
t.Fatalf("expected missing env name in error, got %v", err)
|
t.Fatalf("error = %v, want upstream generation error without credential identities", err)
|
||||||
|
}
|
||||||
|
if !errors.Is(err, promptkit.ErrLLMGenerate) {
|
||||||
|
t.Fatalf("error = %v, want ErrLLMGenerate", err)
|
||||||
|
}
|
||||||
|
var generationErr *promptkit.GenerationError
|
||||||
|
if !errors.As(err, &generationErr) {
|
||||||
|
t.Fatalf("error = %v, want GenerationError", err)
|
||||||
|
}
|
||||||
|
if generationErr.StatusCode() != http.StatusUnauthorized ||
|
||||||
|
generationErr.ProviderType() != "authentication_error" ||
|
||||||
|
generationErr.ProviderCode() != "invalid_api_key" ||
|
||||||
|
generationErr.ProviderMessage() != "authentication failed" {
|
||||||
|
t.Fatalf("GenerationError = %+v, want structured upstream authentication failure", generationErr)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -1983,6 +2018,25 @@ func TestWithProfilesRejectsDuplicateIDs(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestWithProfilesAcceptsDerivedDefinitionWithoutTargetFields(t *testing.T) {
|
||||||
|
_, err := promptkit.NewEngine(promptkit.Config{PromptDir: frameworkPromptDir},
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: " derived-profile ",
|
||||||
|
BaseProfileID: " base-profile ",
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("derived profile should be accepted during construction: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = promptkit.NewEngine(promptkit.Config{PromptDir: frameworkPromptDir},
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{ID: "standalone-profile"}),
|
||||||
|
)
|
||||||
|
if !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||||
|
t.Fatalf("standalone incomplete profile error = %v, want ErrInvalidConfig", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestWithProfilesRejectsInvalidExecutionSettings(t *testing.T) {
|
func TestWithProfilesRejectsInvalidExecutionSettings(t *testing.T) {
|
||||||
type testCase struct {
|
type testCase struct {
|
||||||
name string
|
name string
|
||||||
@@ -2135,6 +2189,7 @@ func TestOpenAICompatibleProfileMapsEveryField(t *testing.T) {
|
|||||||
extraParams := map[string]any{"provider_option": "distinct-extra-params"}
|
extraParams := map[string]any{"provider_option": "distinct-extra-params"}
|
||||||
got := promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
got := promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||||
ID: "distinct-id",
|
ID: "distinct-id",
|
||||||
|
BaseProfileID: "distinct-base",
|
||||||
BackendID: "distinct-backend",
|
BackendID: "distinct-backend",
|
||||||
Endpoint: "https://distinct.example/v1",
|
Endpoint: "https://distinct.example/v1",
|
||||||
Model: "distinct-model",
|
Model: "distinct-model",
|
||||||
@@ -2149,6 +2204,7 @@ func TestOpenAICompatibleProfileMapsEveryField(t *testing.T) {
|
|||||||
})
|
})
|
||||||
want := promptkit.Profile{
|
want := promptkit.Profile{
|
||||||
ID: "distinct-id",
|
ID: "distinct-id",
|
||||||
|
BaseProfileID: "distinct-base",
|
||||||
BackendID: "distinct-backend",
|
BackendID: "distinct-backend",
|
||||||
Endpoint: "https://distinct.example/v1",
|
Endpoint: "https://distinct.example/v1",
|
||||||
Model: "distinct-model",
|
Model: "distinct-model",
|
||||||
@@ -3430,9 +3486,10 @@ extra_params:
|
|||||||
}
|
}
|
||||||
|
|
||||||
type fakeLLMClient struct {
|
type fakeLLMClient struct {
|
||||||
response *promptkit.GenerateResponse
|
response *promptkit.GenerateResponse
|
||||||
err error
|
responses []*promptkit.GenerateResponse
|
||||||
requests []promptkit.GenerateRequest
|
err error
|
||||||
|
requests []promptkit.GenerateRequest
|
||||||
}
|
}
|
||||||
|
|
||||||
type recordingArtifactReader struct {
|
type recordingArtifactReader struct {
|
||||||
@@ -3608,5 +3665,12 @@ func (f *fakeLLMClient) Generate(_ context.Context, req promptkit.GenerateReques
|
|||||||
if f.err != nil {
|
if f.err != nil {
|
||||||
return nil, f.err
|
return nil, f.err
|
||||||
}
|
}
|
||||||
|
if len(f.responses) > 0 {
|
||||||
|
index := len(f.requests) - 1
|
||||||
|
if index >= len(f.responses) {
|
||||||
|
return nil, fmt.Errorf("no response configured for generation %d", index+1)
|
||||||
|
}
|
||||||
|
return f.responses[index], nil
|
||||||
|
}
|
||||||
return f.response, nil
|
return f.response, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -21,16 +21,20 @@ func mapPublicError(err error) error {
|
|||||||
strings.TrimSpace(internalCapacityError.BackendID) != "" {
|
strings.TrimSpace(internalCapacityError.BackendID) != "" {
|
||||||
return &CapacityError{BackendID: internalCapacityError.BackendID}
|
return &CapacityError{BackendID: internalCapacityError.BackendID}
|
||||||
}
|
}
|
||||||
|
publicErr := publicErrorFor(err)
|
||||||
var providerHTTPError *llm.ProviderHTTPError
|
var providerHTTPError *llm.ProviderHTTPError
|
||||||
if errors.As(err, &providerHTTPError) && providerHTTPError != nil {
|
if errors.As(err, &providerHTTPError) && providerHTTPError != nil {
|
||||||
return newGenerationError(
|
generationErr := newGenerationError(
|
||||||
providerHTTPError.StatusCode(),
|
providerHTTPError.StatusCode(),
|
||||||
providerHTTPError.ProviderCode(),
|
providerHTTPError.ProviderCode(),
|
||||||
providerHTTPError.ProviderType(),
|
providerHTTPError.ProviderType(),
|
||||||
providerHTTPError.ProviderMessage(),
|
providerHTTPError.ProviderMessage(),
|
||||||
)
|
)
|
||||||
|
if publicErr != nil && !errors.Is(publicErr, ErrLLMGenerate) {
|
||||||
|
return fmt.Errorf("%w: %w", publicErr, generationErr)
|
||||||
|
}
|
||||||
|
return generationErr
|
||||||
}
|
}
|
||||||
publicErr := publicErrorFor(err)
|
|
||||||
if publicErr == nil {
|
if publicErr == nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,6 +6,7 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -48,3 +49,27 @@ func TestMapPublicErrorTranslatesCapacityError(t *testing.T) {
|
|||||||
t.Fatalf("mapped backend ID changed with source error: %q", publicErr.BackendID)
|
t.Fatalf("mapped backend ID changed with source error: %q", publicErr.BackendID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestMapPublicErrorPreservesValidationAroundGenerationError(t *testing.T) {
|
||||||
|
internalErr := fmt.Errorf(
|
||||||
|
"%w: %w",
|
||||||
|
usecase.ErrValidation,
|
||||||
|
&llm.ProviderHTTPError{},
|
||||||
|
)
|
||||||
|
|
||||||
|
err := mapPublicError(internalErr)
|
||||||
|
if !errors.Is(err, ErrValidation) {
|
||||||
|
t.Fatalf("mapped error=%v, want ErrValidation", err)
|
||||||
|
}
|
||||||
|
if !errors.Is(err, ErrLLMGenerate) {
|
||||||
|
t.Fatalf("mapped error=%v, want ErrLLMGenerate", err)
|
||||||
|
}
|
||||||
|
var generationErr *GenerationError
|
||||||
|
if !errors.As(err, &generationErr) || generationErr == nil {
|
||||||
|
t.Fatalf("mapped error=%v, want GenerationError", err)
|
||||||
|
}
|
||||||
|
var leakedInternalErr *llm.ProviderHTTPError
|
||||||
|
if errors.As(err, &leakedInternalErr) {
|
||||||
|
t.Fatalf("mapped error exposes internal ProviderHTTPError: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -39,6 +39,44 @@ func TestBuiltInGenerationError(t *testing.T) {
|
|||||||
assertGenerationError(t, err, http.StatusServiceUnavailable, "", "", "")
|
assertGenerationError(t, err, http.StatusServiceUnavailable, "", "", "")
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestBuiltInRepairGenerationError(t *testing.T) {
|
||||||
|
const (
|
||||||
|
codeMarker = "repair-code-marker"
|
||||||
|
typeMarker = "repair-type-marker"
|
||||||
|
messageMarker = "repair-message-marker"
|
||||||
|
)
|
||||||
|
calls := 0
|
||||||
|
config := contractConfig(frameworkSchemaDir)
|
||||||
|
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
|
||||||
|
calls++
|
||||||
|
if calls == 1 {
|
||||||
|
body := `{"choices":[{"message":{"content":"not-json"}}]}`
|
||||||
|
return &http.Response{StatusCode: http.StatusOK, ContentLength: int64(len(body)), Body: io.NopCloser(strings.NewReader(body))}, nil
|
||||||
|
}
|
||||||
|
body := `{"error":{"code":"` + codeMarker + `","type":"` + typeMarker + `","message":"` + messageMarker + `"}}`
|
||||||
|
return &http.Response{StatusCode: http.StatusUnprocessableEntity, ContentLength: int64(len(body)), Body: io.NopCloser(strings.NewReader(body))}, nil
|
||||||
|
})}
|
||||||
|
engine, err := promptkit.NewEngine(config)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewEngine: %v", err)
|
||||||
|
}
|
||||||
|
req := generationErrorRunRequest()
|
||||||
|
req.Validation = &promptkit.OutputContract{
|
||||||
|
Format: promptkit.FormatJSON,
|
||||||
|
ValidationMode: promptkit.ValidationJSON,
|
||||||
|
RepairAttempts: 1,
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := engine.Run(context.Background(), req)
|
||||||
|
if result != nil {
|
||||||
|
t.Fatalf("Run result = %#v, want nil", result)
|
||||||
|
}
|
||||||
|
if calls != 2 {
|
||||||
|
t.Fatalf("provider calls = %d, want 2", calls)
|
||||||
|
}
|
||||||
|
assertGenerationError(t, err, http.StatusUnprocessableEntity, codeMarker, typeMarker, messageMarker)
|
||||||
|
}
|
||||||
|
|
||||||
func assertGenerationError(t *testing.T, err error, statusCode int, code, providerType, message string) {
|
func assertGenerationError(t *testing.T, err error, statusCode int, code, providerType, message string) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
|
|
||||||
|
|||||||
@@ -192,6 +192,7 @@ type BackendCapacityPolicy struct {
|
|||||||
// ExecutionProfile describes how and where to execute a model.
|
// ExecutionProfile describes how and where to execute a model.
|
||||||
type ExecutionProfile struct {
|
type ExecutionProfile struct {
|
||||||
ID string `yaml:"id"`
|
ID string `yaml:"id"`
|
||||||
|
BaseProfileID string `yaml:"base_profile"`
|
||||||
BackendID string `yaml:"backend"`
|
BackendID string `yaml:"backend"`
|
||||||
Endpoint string `yaml:"endpoint"`
|
Endpoint string `yaml:"endpoint"`
|
||||||
Model string `yaml:"model"`
|
Model string `yaml:"model"`
|
||||||
|
|||||||
@@ -6,6 +6,8 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const maxOutputRepairAttempts = 3
|
||||||
|
|
||||||
// ValidateOutputContract validates source-neutral output-contract invariants.
|
// ValidateOutputContract validates source-neutral output-contract invariants.
|
||||||
func ValidateOutputContract(contract OutputContract) error {
|
func ValidateOutputContract(contract OutputContract) error {
|
||||||
switch contract.Format {
|
switch contract.Format {
|
||||||
@@ -26,5 +28,11 @@ func ValidateOutputContract(contract OutputContract) error {
|
|||||||
if contract.RepairAttempts < 0 {
|
if contract.RepairAttempts < 0 {
|
||||||
return errors.New("repair_attempts must be greater than or equal to 0")
|
return errors.New("repair_attempts must be greater than or equal to 0")
|
||||||
}
|
}
|
||||||
|
if contract.RepairAttempts > maxOutputRepairAttempts {
|
||||||
|
return fmt.Errorf("repair_attempts must be less than or equal to %d", maxOutputRepairAttempts)
|
||||||
|
}
|
||||||
|
if contract.ValidationMode == ValidationNone && contract.RepairAttempts > 0 {
|
||||||
|
return errors.New("repair_attempts requires basic, json, or json_schema validation")
|
||||||
|
}
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -30,9 +30,28 @@ func TestValidateOutputContract(t *testing.T) {
|
|||||||
}},
|
}},
|
||||||
{name: "empty validation mode", change: func(c *OutputContract) { c.ValidationMode = "" }, wantErr: "validation mode"},
|
{name: "empty validation mode", change: func(c *OutputContract) { c.ValidationMode = "" }, wantErr: "validation mode"},
|
||||||
{name: "unsupported validation mode", change: func(c *OutputContract) { c.ValidationMode = ValidationMode("unknown") }, wantErr: "validation mode"},
|
{name: "unsupported validation mode", change: func(c *OutputContract) { c.ValidationMode = ValidationMode("unknown") }, wantErr: "validation mode"},
|
||||||
{name: "negative repair attempts", change: func(c *OutputContract) { c.RepairAttempts = -1 }, wantErr: "repair_attempts"},
|
{name: "negative repair attempts", change: func(c *OutputContract) {
|
||||||
|
c.ValidationMode = ValidationBasic
|
||||||
|
c.RepairAttempts = -1
|
||||||
|
}, wantErr: "repair_attempts"},
|
||||||
{name: "zero repair attempts", change: func(c *OutputContract) { c.RepairAttempts = 0 }},
|
{name: "zero repair attempts", change: func(c *OutputContract) { c.RepairAttempts = 0 }},
|
||||||
{name: "positive repair attempts", change: func(c *OutputContract) { c.RepairAttempts = 1 }},
|
{name: "one repair attempt", change: func(c *OutputContract) {
|
||||||
|
c.ValidationMode = ValidationBasic
|
||||||
|
c.RepairAttempts = 1
|
||||||
|
}},
|
||||||
|
{name: "maximum repair attempts", change: func(c *OutputContract) {
|
||||||
|
c.ValidationMode = ValidationJSON
|
||||||
|
c.RepairAttempts = 3
|
||||||
|
}},
|
||||||
|
{name: "too many repair attempts", change: func(c *OutputContract) {
|
||||||
|
c.ValidationMode = ValidationJSONSchema
|
||||||
|
c.SchemaPath = "schema.json"
|
||||||
|
c.RepairAttempts = 4
|
||||||
|
}, wantErr: "repair_attempts"},
|
||||||
|
{name: "none validation with repair attempts", change: func(c *OutputContract) {
|
||||||
|
c.ValidationMode = ValidationNone
|
||||||
|
c.RepairAttempts = 1
|
||||||
|
}, wantErr: "repair_attempts"},
|
||||||
{name: "json schema empty path", change: func(c *OutputContract) {
|
{name: "json schema empty path", change: func(c *OutputContract) {
|
||||||
c.ValidationMode = ValidationJSONSchema
|
c.ValidationMode = ValidationJSONSchema
|
||||||
c.SchemaPath = ""
|
c.SchemaPath = ""
|
||||||
|
|||||||
@@ -133,13 +133,18 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
|||||||
return nil, fmt.Errorf("%w: failed to create request: %v", ErrRequestFailed, err)
|
return nil, fmt.Errorf("%w: failed to create request: %v", ErrRequestFailed, err)
|
||||||
}
|
}
|
||||||
httpReq.Header.Set("Content-Type", "application/json")
|
httpReq.Header.Set("Content-Type", "application/json")
|
||||||
if apiKey := strings.TrimSpace(req.Target.APIKey); apiKey != "" {
|
apiKey := strings.TrimSpace(req.Target.APIKey)
|
||||||
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
|
envName := strings.TrimSpace(req.Target.APIKeyEnv)
|
||||||
} else if envName := strings.TrimSpace(req.Target.APIKeyEnv); envName != "" {
|
if apiKey == "" && envName != "" {
|
||||||
apiKey := strings.TrimSpace(os.Getenv(envName))
|
apiKey = strings.TrimSpace(os.Getenv(envName))
|
||||||
if apiKey == "" {
|
}
|
||||||
|
if apiKey == "" && req.Target.APIKeyRequired {
|
||||||
|
if envName != "" {
|
||||||
return nil, fmt.Errorf("%w: api key environment variable %q is not set", ErrInvalidRequest, envName)
|
return nil, fmt.Errorf("%w: api key environment variable %q is not set", ErrInvalidRequest, envName)
|
||||||
}
|
}
|
||||||
|
return nil, fmt.Errorf("%w: api key is required", ErrInvalidRequest)
|
||||||
|
}
|
||||||
|
if apiKey != "" {
|
||||||
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
|
httpReq.Header.Set("Authorization", "Bearer "+apiKey)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -174,12 +179,12 @@ func (c *OpenAICompatibleClient) Generate(ctx context.Context, req domain.Genera
|
|||||||
return nil, fmt.Errorf("%w: no choices returned", ErrMalformedResponse)
|
return nil, fmt.Errorf("%w: no choices returned", ErrMalformedResponse)
|
||||||
}
|
}
|
||||||
content := wireResp.Choices[0].Message.Content
|
content := wireResp.Choices[0].Message.Content
|
||||||
if content == "" {
|
if content == nil {
|
||||||
return nil, fmt.Errorf("%w: first choice has empty message content", ErrMalformedResponse)
|
return nil, fmt.Errorf("%w: first choice has missing message content", ErrMalformedResponse)
|
||||||
}
|
}
|
||||||
|
|
||||||
return &domain.GenerateResponse{
|
return &domain.GenerateResponse{
|
||||||
Content: content,
|
Content: *content,
|
||||||
Usage: domain.TokenUsage{
|
Usage: domain.TokenUsage{
|
||||||
PromptTokens: wireResp.Usage.PromptTokens,
|
PromptTokens: wireResp.Usage.PromptTokens,
|
||||||
CompletionTokens: wireResp.Usage.CompletionTokens,
|
CompletionTokens: wireResp.Usage.CompletionTokens,
|
||||||
@@ -374,8 +379,8 @@ type openAICacheControl struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type openAIChatResponseMessage struct {
|
type openAIChatResponseMessage struct {
|
||||||
Role string `json:"role"`
|
Role string `json:"role"`
|
||||||
Content string `json:"content"`
|
Content *string `json:"content"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type openAIChatResponse struct {
|
type openAIChatResponse struct {
|
||||||
|
|||||||
@@ -501,36 +501,61 @@ func checkCompleteRequestAndResponseMapping(t *testing.T) {
|
|||||||
|
|
||||||
func TestOpenAICompatibleClientAuthentication(t *testing.T) {
|
func TestOpenAICompatibleClientAuthentication(t *testing.T) {
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
configureEnv func(*testing.T)
|
configureEnv func(*testing.T)
|
||||||
target domain.ExecutionTarget
|
target domain.ExecutionTarget
|
||||||
wantAuth string
|
wantAuthorization string
|
||||||
wantErr error
|
wantError error
|
||||||
wantCallCount int
|
wantCallCount int
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "direct key takes precedence over environment",
|
name: "direct key takes precedence over environment",
|
||||||
configureEnv: func(t *testing.T) {
|
configureEnv: func(t *testing.T) {
|
||||||
t.Setenv("PROMPTKIT_TEST_API_KEY", "env-key")
|
t.Setenv("PROMPTKIT_TEST_API_KEY", " env-key ")
|
||||||
},
|
},
|
||||||
target: domain.ExecutionTarget{
|
target: domain.ExecutionTarget{
|
||||||
APIKeyEnv: "PROMPTKIT_TEST_API_KEY",
|
APIKeyEnv: "PROMPTKIT_TEST_API_KEY",
|
||||||
APIKey: "direct-llm-key",
|
APIKey: " direct-llm-key ",
|
||||||
},
|
},
|
||||||
wantAuth: "Bearer direct-llm-key",
|
wantAuthorization: "Bearer direct-llm-key",
|
||||||
wantCallCount: 1,
|
wantCallCount: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "environment key supplies authorization",
|
||||||
|
configureEnv: func(t *testing.T) {
|
||||||
|
t.Setenv("PROMPTKIT_TEST_API_KEY", " env-key ")
|
||||||
|
},
|
||||||
|
target: domain.ExecutionTarget{APIKeyEnv: "PROMPTKIT_TEST_API_KEY"},
|
||||||
|
wantAuthorization: "Bearer env-key",
|
||||||
|
wantCallCount: 1,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "no key omits authorization",
|
name: "no key omits authorization",
|
||||||
wantCallCount: 1,
|
wantCallCount: 1,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "missing environment key fails before transport",
|
name: "optional missing environment omits authorization",
|
||||||
configureEnv: func(t *testing.T) {
|
configureEnv: func(t *testing.T) {
|
||||||
t.Setenv("PROMPTKIT_MISSING_KEY", "")
|
t.Setenv("PROMPTKIT_MISSING_KEY", "")
|
||||||
},
|
},
|
||||||
target: domain.ExecutionTarget{APIKeyEnv: "PROMPTKIT_MISSING_KEY"},
|
target: domain.ExecutionTarget{APIKeyEnv: "PROMPTKIT_MISSING_KEY"},
|
||||||
wantErr: ErrInvalidRequest,
|
wantCallCount: 1,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "required missing environment fails before transport",
|
||||||
|
configureEnv: func(t *testing.T) {
|
||||||
|
t.Setenv("PROMPTKIT_MISSING_KEY", "")
|
||||||
|
},
|
||||||
|
target: domain.ExecutionTarget{
|
||||||
|
APIKeyEnv: "PROMPTKIT_MISSING_KEY",
|
||||||
|
APIKeyRequired: true,
|
||||||
|
},
|
||||||
|
wantError: ErrInvalidRequest,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "required target without source fails before transport",
|
||||||
|
target: domain.ExecutionTarget{APIKeyRequired: true},
|
||||||
|
wantError: ErrInvalidRequest,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -545,9 +570,9 @@ func TestOpenAICompatibleClientAuthentication(t *testing.T) {
|
|||||||
request.Target = tc.target
|
request.Target = tc.target
|
||||||
|
|
||||||
_, err := client.Generate(context.Background(), request)
|
_, err := client.Generate(context.Background(), request)
|
||||||
if tc.wantErr != nil {
|
if tc.wantError != nil {
|
||||||
if !errors.Is(err, tc.wantErr) {
|
if !errors.Is(err, tc.wantError) {
|
||||||
t.Fatalf("error = %v, want %v", err, tc.wantErr)
|
t.Fatalf("error = %v, want %v", err, tc.wantError)
|
||||||
}
|
}
|
||||||
} else if err != nil {
|
} else if err != nil {
|
||||||
t.Fatalf("generate: %v", err)
|
t.Fatalf("generate: %v", err)
|
||||||
@@ -556,8 +581,13 @@ func TestOpenAICompatibleClientAuthentication(t *testing.T) {
|
|||||||
t.Fatalf("provider calls = %d, want %d", got, tc.wantCallCount)
|
t.Fatalf("provider calls = %d, want %d", got, tc.wantCallCount)
|
||||||
}
|
}
|
||||||
if tc.wantCallCount == 1 {
|
if tc.wantCallCount == 1 {
|
||||||
if got := provider.lastRequest(t).header.Get("Authorization"); got != tc.wantAuth {
|
values := provider.lastRequest(t).header.Values("Authorization")
|
||||||
t.Fatalf("Authorization = %q, want %q", got, tc.wantAuth)
|
if tc.wantAuthorization == "" {
|
||||||
|
if len(values) != 0 {
|
||||||
|
t.Fatalf("Authorization values = %q, want absent", values)
|
||||||
|
}
|
||||||
|
} else if len(values) != 1 || values[0] != tc.wantAuthorization {
|
||||||
|
t.Fatalf("Authorization values = %q, want [%q]", values, tc.wantAuthorization)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
@@ -735,6 +765,7 @@ func TestOpenAICompatibleClientResponseFraming(t *testing.T) {
|
|||||||
run func(*testing.T)
|
run func(*testing.T)
|
||||||
}{
|
}{
|
||||||
{name: "usage mapping", run: checkCacheUsageMapping},
|
{name: "usage mapping", run: checkCacheUsageMapping},
|
||||||
|
{name: "content presence", run: checkContentPresence},
|
||||||
{name: "common response failures", run: checkCommonResponseFailures},
|
{name: "common response failures", run: checkCommonResponseFailures},
|
||||||
{name: "successful response byte boundary", run: checkSuccessfulResponseByteBoundary},
|
{name: "successful response byte boundary", run: checkSuccessfulResponseByteBoundary},
|
||||||
{name: "continuing oversized response", run: checkContinuingOversizedResponse},
|
{name: "continuing oversized response", run: checkContinuingOversizedResponse},
|
||||||
@@ -746,6 +777,53 @@ func TestOpenAICompatibleClientResponseFraming(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func checkContentPresence(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
content string
|
||||||
|
}{
|
||||||
|
{name: "explicit empty string", content: ""},
|
||||||
|
{name: "whitespace string", content: " \n\t "},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
provider := newRecordingProvider(t)
|
||||||
|
provider.respond(http.StatusOK, `{
|
||||||
|
"choices": [{"message": {"content": `+strconv.Quote(tc.content)+`}}],
|
||||||
|
"usage": {
|
||||||
|
"prompt_tokens": 10,
|
||||||
|
"completion_tokens": 20,
|
||||||
|
"total_tokens": 30,
|
||||||
|
"prompt_tokens_details": {"cached_tokens": 4},
|
||||||
|
"cache_write_tokens": 5
|
||||||
|
}
|
||||||
|
}`)
|
||||||
|
client := newProviderClient(t, provider, OpenAICompatibleConfig{Model: "model"})
|
||||||
|
|
||||||
|
response, err := client.Generate(context.Background(), ordinaryGenerateRequest())
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("generate: %v", err)
|
||||||
|
}
|
||||||
|
if response == nil {
|
||||||
|
t.Fatal("expected response")
|
||||||
|
}
|
||||||
|
if response.Content != tc.content {
|
||||||
|
t.Fatalf("content = %q, want %q", response.Content, tc.content)
|
||||||
|
}
|
||||||
|
if response.Usage != (domain.TokenUsage{
|
||||||
|
PromptTokens: 10,
|
||||||
|
CompletionTokens: 20,
|
||||||
|
TotalTokens: 30,
|
||||||
|
CachedTokens: 4,
|
||||||
|
CacheWriteTokens: 5,
|
||||||
|
}) {
|
||||||
|
t.Fatalf("usage = %+v", response.Usage)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func checkCacheUsageMapping(t *testing.T) {
|
func checkCacheUsageMapping(t *testing.T) {
|
||||||
provider := newRecordingProvider(t)
|
provider := newRecordingProvider(t)
|
||||||
provider.respond(http.StatusOK, `{
|
provider.respond(http.StatusOK, `{
|
||||||
@@ -1099,6 +1177,9 @@ func checkCommonResponseFailures(t *testing.T) {
|
|||||||
},
|
},
|
||||||
{name: "invalid JSON", statusCode: http.StatusOK, body: `{not valid json`, wantErr: ErrMalformedResponse},
|
{name: "invalid JSON", statusCode: http.StatusOK, body: `{not valid json`, wantErr: ErrMalformedResponse},
|
||||||
{name: "missing choices", statusCode: http.StatusOK, body: `{"choices": []}`, wantErr: ErrMalformedResponse},
|
{name: "missing choices", statusCode: http.StatusOK, body: `{"choices": []}`, wantErr: ErrMalformedResponse},
|
||||||
|
{name: "missing content", statusCode: http.StatusOK, body: `{"choices": [{"message": {}}]}`, wantErr: ErrMalformedResponse},
|
||||||
|
{name: "null content", statusCode: http.StatusOK, body: `{"choices": [{"message": {"content": null}}]}`, wantErr: ErrMalformedResponse},
|
||||||
|
{name: "non-string content", statusCode: http.StatusOK, body: `{"choices": [{"message": {"content": 1}}]}`, wantErr: ErrMalformedResponse},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
|
|||||||
47
internal/profile/definition.go
Normal file
47
internal/profile/definition.go
Normal file
@@ -0,0 +1,47 @@
|
|||||||
|
package profile
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NormalizeAndValidateDefinition normalizes and validates one source-local
|
||||||
|
// profile definition without resolving a base profile.
|
||||||
|
func NormalizeAndValidateDefinition(profile *domain.ExecutionProfile) error {
|
||||||
|
if profile == nil {
|
||||||
|
return errors.New("profile is required")
|
||||||
|
}
|
||||||
|
|
||||||
|
profile.ID = strings.TrimSpace(profile.ID)
|
||||||
|
profile.BaseProfileID = strings.TrimSpace(profile.BaseProfileID)
|
||||||
|
profile.BackendID = strings.TrimSpace(profile.BackendID)
|
||||||
|
profile.Endpoint = strings.TrimSpace(profile.Endpoint)
|
||||||
|
|
||||||
|
if profile.ID == "" {
|
||||||
|
return errors.New("id is required")
|
||||||
|
}
|
||||||
|
if profile.Endpoint != "" {
|
||||||
|
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(profile.Endpoint)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
profile.Endpoint = endpoint
|
||||||
|
}
|
||||||
|
if profile.BaseProfileID == "" {
|
||||||
|
if profile.BackendID == "" && profile.Endpoint == "" {
|
||||||
|
return errors.New("backend or endpoint is required")
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(profile.Model) == "" {
|
||||||
|
return errors.New("model is required")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{
|
||||||
|
Temperature: profile.Temperature,
|
||||||
|
MaxTokens: profile.MaxTokens,
|
||||||
|
TopP: profile.TopP,
|
||||||
|
TimeoutSeconds: profile.TimeoutSeconds,
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -127,12 +127,11 @@ func loadProfile(ctx context.Context, fsys fs.FS, root string, id string) (*doma
|
|||||||
if prof.ID != id {
|
if prof.ID != id {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
prof.BackendID = strings.TrimSpace(prof.BackendID)
|
|
||||||
prof.ExtraParams, err = jsonvalue.CopyMap(prof.ExtraParams)
|
prof.ExtraParams, err = jsonvalue.CopyMap(prof.ExtraParams)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidProfile, relPath, err)
|
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidProfile, relPath, err)
|
||||||
}
|
}
|
||||||
if err := normalizeAndValidateProfile(prof); err != nil {
|
if err := NormalizeAndValidateDefinition(prof); err != nil {
|
||||||
if errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
if errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
||||||
return nil, fmt.Errorf("%w: %s", err, relPath)
|
return nil, fmt.Errorf("%w: %s", err, relPath)
|
||||||
}
|
}
|
||||||
@@ -255,30 +254,3 @@ func requireYAMLStreamEnd(decoder *yaml.Decoder) error {
|
|||||||
}
|
}
|
||||||
return errors.New("profile file must contain exactly one YAML document")
|
return errors.New("profile file must contain exactly one YAML document")
|
||||||
}
|
}
|
||||||
|
|
||||||
func normalizeAndValidateProfile(p *domain.ExecutionProfile) error {
|
|
||||||
if strings.TrimSpace(p.ID) == "" {
|
|
||||||
return errors.New("id is required")
|
|
||||||
}
|
|
||||||
p.Endpoint = strings.TrimSpace(p.Endpoint)
|
|
||||||
if strings.TrimSpace(p.BackendID) == "" && p.Endpoint == "" {
|
|
||||||
return errors.New("backend or endpoint is required")
|
|
||||||
}
|
|
||||||
if p.Endpoint != "" {
|
|
||||||
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(p.Endpoint)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
p.Endpoint = endpoint
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(p.Model) == "" {
|
|
||||||
return errors.New("model is required")
|
|
||||||
}
|
|
||||||
|
|
||||||
return domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{
|
|
||||||
Temperature: p.Temperature,
|
|
||||||
MaxTokens: p.MaxTokens,
|
|
||||||
TopP: p.TopP,
|
|
||||||
TimeoutSeconds: p.TimeoutSeconds,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -858,6 +858,107 @@ top_p: .inf
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestProfileRepositoriesValidateDerivedDefinitions(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
files map[string]string
|
||||||
|
wantError error
|
||||||
|
wantBaseID string
|
||||||
|
wantProfile bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "alias is locally valid and normalizes base id",
|
||||||
|
files: map[string]string{"alias.yaml": `
|
||||||
|
id: selected-profile
|
||||||
|
base_profile: " base-profile "
|
||||||
|
`},
|
||||||
|
wantBaseID: "base-profile",
|
||||||
|
wantProfile: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "derived endpoint remains valid",
|
||||||
|
files: map[string]string{"invalid.yaml": `
|
||||||
|
id: selected-profile
|
||||||
|
base_profile: base-profile
|
||||||
|
endpoint: /v1
|
||||||
|
`},
|
||||||
|
wantError: ErrInvalidProfile,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "derived settings remain valid",
|
||||||
|
files: map[string]string{"invalid.yaml": `
|
||||||
|
id: selected-profile
|
||||||
|
base_profile: base-profile
|
||||||
|
top_p: 1.1
|
||||||
|
`},
|
||||||
|
wantError: ErrInvalidProfile,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "derived extra params remain valid",
|
||||||
|
files: map[string]string{"invalid.yaml": `
|
||||||
|
id: selected-profile
|
||||||
|
base_profile: base-profile
|
||||||
|
extra_params:
|
||||||
|
timestamp: 2026-08-11T12:34:56Z
|
||||||
|
`},
|
||||||
|
wantError: ErrInvalidProfile,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "derived raw key remains prohibited",
|
||||||
|
files: map[string]string{"invalid.yaml": `
|
||||||
|
id: selected-profile
|
||||||
|
base_profile: base-profile
|
||||||
|
api_key: secret
|
||||||
|
`},
|
||||||
|
wantError: ErrRawAPIKeyNotAllowed,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "derived duplicate id remains invalid",
|
||||||
|
files: map[string]string{
|
||||||
|
"first.yaml": "id: selected-profile\nbase_profile: first-base\n",
|
||||||
|
"second.yaml": "id: selected-profile\nbase_profile: second-base\n",
|
||||||
|
},
|
||||||
|
wantError: ErrInvalidProfile,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "derived extra document remains invalid",
|
||||||
|
files: map[string]string{"invalid.yaml": `
|
||||||
|
id: selected-profile
|
||||||
|
base_profile: base-profile
|
||||||
|
---
|
||||||
|
id: other
|
||||||
|
`},
|
||||||
|
wantError: ErrInvalidYAML,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "standalone profile remains complete",
|
||||||
|
files: map[string]string{"invalid.yaml": "id: selected-profile\n"},
|
||||||
|
wantError: ErrInvalidProfile,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, source := range profileRepositorySources() {
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(source.name+"/"+tc.name, func(t *testing.T) {
|
||||||
|
repo := source.newRepository(t, tc.files)
|
||||||
|
got, err := repo.GetProfile(context.Background(), "selected-profile")
|
||||||
|
if tc.wantError != nil {
|
||||||
|
if !errors.Is(err, tc.wantError) {
|
||||||
|
t.Fatalf("error = %v, want %v", err, tc.wantError)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil || !tc.wantProfile {
|
||||||
|
t.Fatalf("profile = %+v, error = %v, want valid derived definition", got, err)
|
||||||
|
}
|
||||||
|
if got.BaseProfileID != tc.wantBaseID {
|
||||||
|
t.Fatalf("BaseProfileID = %q, want %q", got.BaseProfileID, tc.wantBaseID)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestOverlayRepository(t *testing.T) {
|
func TestOverlayRepository(t *testing.T) {
|
||||||
ctx := context.Background()
|
ctx := context.Background()
|
||||||
primaryProfile := &domain.ExecutionProfile{ID: "shared", Endpoint: "http://primary", Model: "primary"}
|
primaryProfile := &domain.ExecutionProfile{ID: "shared", Endpoint: "http://primary", Model: "primary"}
|
||||||
|
|||||||
160
internal/profile/resolving_repository.go
Normal file
160
internal/profile/resolving_repository.go
Normal file
@@ -0,0 +1,160 @@
|
|||||||
|
package profile
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||||
|
)
|
||||||
|
|
||||||
|
const maximumProfileChainLength = 32
|
||||||
|
|
||||||
|
type resolvingRepository struct {
|
||||||
|
source Repository
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewResolvingRepository resolves inherited profile definitions from source.
|
||||||
|
func NewResolvingRepository(source Repository) Repository {
|
||||||
|
return &resolvingRepository{source: source}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *resolvingRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||||
|
if r == nil || r.source == nil {
|
||||||
|
return nil, fmt.Errorf("%w: profile repository is required", ErrInvalidProfile)
|
||||||
|
}
|
||||||
|
requestedID := strings.TrimSpace(id)
|
||||||
|
if requestedID == "" {
|
||||||
|
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile)
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
profile, err := r.getRawProfile(ctx, requestedID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if profile == nil {
|
||||||
|
return nil, fmt.Errorf("%w: selected profile %q is nil", ErrInvalidProfile, requestedID)
|
||||||
|
}
|
||||||
|
|
||||||
|
chain := []*domain.ExecutionProfile{profile}
|
||||||
|
chainIDs := []string{requestedID}
|
||||||
|
visited := map[string]struct{}{requestedID: {}}
|
||||||
|
current := profile
|
||||||
|
|
||||||
|
for {
|
||||||
|
baseID := strings.TrimSpace(current.BaseProfileID)
|
||||||
|
if baseID == "" {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if _, seen := visited[baseID]; seen {
|
||||||
|
return nil, fmt.Errorf("%w: profile inheritance cycle %s", ErrInvalidProfile, joinProfileChain(chainIDs, baseID))
|
||||||
|
}
|
||||||
|
if len(chain) >= maximumProfileChainLength {
|
||||||
|
return nil, fmt.Errorf("%w: profile inheritance chain exceeds %d profiles: %s", ErrInvalidProfile, maximumProfileChainLength, joinProfileChain(chainIDs, baseID))
|
||||||
|
}
|
||||||
|
|
||||||
|
base, err := r.getRawProfile(ctx, baseID)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, ErrProfileNotFound) {
|
||||||
|
return nil, fmt.Errorf("%w: base profile %q is missing in chain %s", ErrInvalidProfile, baseID, joinProfileChain(chainIDs, baseID))
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("%w: failed to load base profile %q in chain %s: %w", ErrInvalidProfile, baseID, joinProfileChain(chainIDs, baseID), err)
|
||||||
|
}
|
||||||
|
if base == nil {
|
||||||
|
return nil, fmt.Errorf("%w: base profile %q is nil in chain %s", ErrInvalidProfile, baseID, joinProfileChain(chainIDs, baseID))
|
||||||
|
}
|
||||||
|
|
||||||
|
chain = append(chain, base)
|
||||||
|
chainIDs = append(chainIDs, baseID)
|
||||||
|
visited[baseID] = struct{}{}
|
||||||
|
current = base
|
||||||
|
}
|
||||||
|
|
||||||
|
resolved, err := mergeProfileChain(chain)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: resolved profile chain %s: %w", ErrInvalidProfile, strings.Join(chainIDs, " -> "), err)
|
||||||
|
}
|
||||||
|
if err := validateResolvedProfile(resolved); err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: resolved profile chain %s: %w", ErrInvalidProfile, strings.Join(chainIDs, " -> "), err)
|
||||||
|
}
|
||||||
|
return resolved, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *resolvingRepository) getRawProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||||
|
profile, err := r.source.GetProfile(ctx, id)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return profile, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func joinProfileChain(chain []string, next string) string {
|
||||||
|
return strings.Join(append(append([]string(nil), chain...), next), " -> ")
|
||||||
|
}
|
||||||
|
|
||||||
|
func mergeProfileChain(chain []*domain.ExecutionProfile) (*domain.ExecutionProfile, error) {
|
||||||
|
resolved := &domain.ExecutionProfile{ID: chain[0].ID}
|
||||||
|
for index := len(chain) - 1; index >= 0; index-- {
|
||||||
|
definition := chain[index]
|
||||||
|
if strings.TrimSpace(definition.BackendID) != "" {
|
||||||
|
resolved.BackendID = definition.BackendID
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(definition.Endpoint) != "" {
|
||||||
|
resolved.Endpoint = definition.Endpoint
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(definition.Model) != "" {
|
||||||
|
resolved.Model = definition.Model
|
||||||
|
}
|
||||||
|
if definition.Temperature != 0 {
|
||||||
|
resolved.Temperature = definition.Temperature
|
||||||
|
}
|
||||||
|
if definition.MaxTokens != 0 {
|
||||||
|
resolved.MaxTokens = definition.MaxTokens
|
||||||
|
}
|
||||||
|
if definition.TopP != 0 {
|
||||||
|
resolved.TopP = definition.TopP
|
||||||
|
}
|
||||||
|
if definition.TimeoutSeconds != 0 {
|
||||||
|
resolved.TimeoutSeconds = definition.TimeoutSeconds
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(definition.ServiceTier) != "" {
|
||||||
|
resolved.ServiceTier = definition.ServiceTier
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(definition.ReasoningEffort) != "" {
|
||||||
|
resolved.ReasoningEffort = definition.ReasoningEffort
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(definition.APIKeyEnv) != "" {
|
||||||
|
resolved.APIKeyEnv = definition.APIKeyEnv
|
||||||
|
}
|
||||||
|
resolved.APIKeyRequired = resolved.APIKeyRequired || definition.APIKeyRequired
|
||||||
|
if len(definition.ExtraParams) != 0 {
|
||||||
|
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
resolved.ExtraParams = extraParams
|
||||||
|
}
|
||||||
|
}
|
||||||
|
resolved.ID = chain[0].ID
|
||||||
|
resolved.BaseProfileID = ""
|
||||||
|
return resolved, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateResolvedProfile(profile *domain.ExecutionProfile) error {
|
||||||
|
if profile == nil {
|
||||||
|
return errors.New("resolved profile is required")
|
||||||
|
}
|
||||||
|
profile.BaseProfileID = ""
|
||||||
|
return NormalizeAndValidateDefinition(profile)
|
||||||
|
}
|
||||||
418
internal/profile/resolving_repository_test.go
Normal file
418
internal/profile/resolving_repository_test.go
Normal file
@@ -0,0 +1,418 @@
|
|||||||
|
package profile
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io/fs"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestResolvingRepositoryMergesProfileChain(t *testing.T) {
|
||||||
|
repo := &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"leaf": {
|
||||||
|
ID: "leaf",
|
||||||
|
BaseProfileID: "middle",
|
||||||
|
BackendID: "leaf-backend",
|
||||||
|
TopP: 0.8,
|
||||||
|
TimeoutSeconds: 45,
|
||||||
|
ReasoningEffort: "high",
|
||||||
|
},
|
||||||
|
"middle": {
|
||||||
|
ID: "middle",
|
||||||
|
BaseProfileID: "root",
|
||||||
|
Endpoint: "https://middle.example/v1",
|
||||||
|
Model: "middle-model",
|
||||||
|
MaxTokens: 256,
|
||||||
|
APIKeyEnv: "MIDDLE_API_KEY",
|
||||||
|
APIKeyRequired: true,
|
||||||
|
ExtraParams: map[string]any{"middle": map[string]any{"value": "middle"}},
|
||||||
|
},
|
||||||
|
"root": {
|
||||||
|
ID: "root",
|
||||||
|
BackendID: "root-backend",
|
||||||
|
Endpoint: "https://root.example/v1",
|
||||||
|
Model: "root-model",
|
||||||
|
Temperature: 0.3,
|
||||||
|
ServiceTier: "priority",
|
||||||
|
ExtraParams: map[string]any{"root": "value"},
|
||||||
|
},
|
||||||
|
}}
|
||||||
|
|
||||||
|
got, err := NewResolvingRepository(repo).GetProfile(context.Background(), "leaf")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolve profile: %v", err)
|
||||||
|
}
|
||||||
|
want := &domain.ExecutionProfile{
|
||||||
|
ID: "leaf",
|
||||||
|
BackendID: "leaf-backend",
|
||||||
|
Endpoint: "https://middle.example/v1",
|
||||||
|
Model: "middle-model",
|
||||||
|
Temperature: 0.3,
|
||||||
|
MaxTokens: 256,
|
||||||
|
TopP: 0.8,
|
||||||
|
TimeoutSeconds: 45,
|
||||||
|
ServiceTier: "priority",
|
||||||
|
ReasoningEffort: "high",
|
||||||
|
APIKeyEnv: "MIDDLE_API_KEY",
|
||||||
|
APIKeyRequired: true,
|
||||||
|
ExtraParams: map[string]any{"middle": map[string]any{"value": "middle"}},
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("resolved profile:\n got %#v\nwant %#v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvingRepositoryRejectsMissingSourceAndProfileID(t *testing.T) {
|
||||||
|
if _, err := NewResolvingRepository(nil).GetProfile(context.Background(), "profile"); !errors.Is(err, ErrInvalidProfile) {
|
||||||
|
t.Fatalf("nil source error = %v, want ErrInvalidProfile", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
repo := &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{}}
|
||||||
|
if _, err := NewResolvingRepository(repo).GetProfile(context.Background(), " \t "); !errors.Is(err, ErrInvalidProfile) {
|
||||||
|
t.Fatalf("blank id error = %v, want ErrInvalidProfile", err)
|
||||||
|
}
|
||||||
|
if got := repo.callCount(" "); got != 0 {
|
||||||
|
t.Fatalf("blank id looked up source %d times", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvingRepositoryCopiesExtraParams(t *testing.T) {
|
||||||
|
baseParams := map[string]any{"nested": map[string]any{"value": "base"}}
|
||||||
|
repo := &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"child": {ID: "child", BaseProfileID: "base"},
|
||||||
|
"base": {
|
||||||
|
ID: "base",
|
||||||
|
Endpoint: "https://base.example/v1",
|
||||||
|
Model: "model",
|
||||||
|
ExtraParams: baseParams,
|
||||||
|
},
|
||||||
|
}}
|
||||||
|
resolver := NewResolvingRepository(repo)
|
||||||
|
|
||||||
|
first, err := resolver.GetProfile(context.Background(), "child")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolve inherited map: %v", err)
|
||||||
|
}
|
||||||
|
first.ExtraParams["nested"].(map[string]any)["value"] = "mutated"
|
||||||
|
second, err := resolver.GetProfile(context.Background(), "child")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolve inherited map again: %v", err)
|
||||||
|
}
|
||||||
|
if got := second.ExtraParams["nested"].(map[string]any)["value"]; got != "base" {
|
||||||
|
t.Fatalf("later result retained mutation: %v", got)
|
||||||
|
}
|
||||||
|
if got := baseParams["nested"].(map[string]any)["value"]; got != "base" {
|
||||||
|
t.Fatalf("source map retained mutation: %v", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
repo.set("child", &domain.ExecutionProfile{
|
||||||
|
ID: "child",
|
||||||
|
BaseProfileID: "base",
|
||||||
|
ExtraParams: map[string]any{"child": "replacement"},
|
||||||
|
})
|
||||||
|
replaced, err := resolver.GetProfile(context.Background(), "child")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolve replacement map: %v", err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(replaced.ExtraParams, map[string]any{"child": "replacement"}) {
|
||||||
|
t.Fatalf("extra params = %#v, want complete child replacement", replaced.ExtraParams)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvingRepositoryUsesRawOverlayForEachLookup(t *testing.T) {
|
||||||
|
leafSource := NewFSRepository(profileTestFS(map[string]string{
|
||||||
|
"leaf.yaml": "id: leaf\nbase_profile: base\n",
|
||||||
|
}), ".")
|
||||||
|
fallback := NewFSRepository(profileTestFS(map[string]string{
|
||||||
|
"base.yaml": "id: base\nendpoint: https://fallback.example/v1\nmodel: fallback-model\n",
|
||||||
|
}), ".")
|
||||||
|
overlay := NewOverlayRepository(leafSource, fallback)
|
||||||
|
resolver := NewResolvingRepository(overlay)
|
||||||
|
|
||||||
|
got, err := resolver.GetProfile(context.Background(), "leaf")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolve fallback base: %v", err)
|
||||||
|
}
|
||||||
|
if got.Model != "fallback-model" {
|
||||||
|
t.Fatalf("fallback base model = %q", got.Model)
|
||||||
|
}
|
||||||
|
|
||||||
|
shadowing := NewOverlayRepository(NewFSRepository(profileTestFS(map[string]string{
|
||||||
|
"leaf.yaml": "id: leaf\nbase_profile: base\n",
|
||||||
|
"base.yaml": "id: base\nendpoint: https://primary.example/v1\nmodel: primary-model\n",
|
||||||
|
}), "."), fallback)
|
||||||
|
got, err = NewResolvingRepository(shadowing).GetProfile(context.Background(), "leaf")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolve shadowed base: %v", err)
|
||||||
|
}
|
||||||
|
if got.Model != "primary-model" || got.Endpoint != "https://primary.example/v1" {
|
||||||
|
t.Fatalf("shadowed base = %+v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvingRepositoryReportsSafetyAndSourceErrors(t *testing.T) {
|
||||||
|
sourceErr := errors.New("source failure")
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
repo *resolvingTestRepository
|
||||||
|
id string
|
||||||
|
want []error
|
||||||
|
wantNot error
|
||||||
|
contains []string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "missing selected profile preserves not found",
|
||||||
|
repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{}},
|
||||||
|
id: "missing",
|
||||||
|
want: []error{ErrProfileNotFound},
|
||||||
|
wantNot: ErrInvalidProfile,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing base is invalid but not not found",
|
||||||
|
repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"leaf": {ID: "leaf", BaseProfileID: "missing"},
|
||||||
|
}},
|
||||||
|
id: "leaf",
|
||||||
|
want: []error{ErrInvalidProfile},
|
||||||
|
wantNot: ErrProfileNotFound,
|
||||||
|
contains: []string{"missing", "leaf -> missing"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "direct cycle",
|
||||||
|
repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"a": {ID: "a", BaseProfileID: "a"},
|
||||||
|
}},
|
||||||
|
id: "a",
|
||||||
|
want: []error{ErrInvalidProfile},
|
||||||
|
contains: []string{"a -> a"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "indirect cycle",
|
||||||
|
repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"a": {ID: "a", BaseProfileID: "b"},
|
||||||
|
"b": {ID: "b", BaseProfileID: "c"},
|
||||||
|
"c": {ID: "c", BaseProfileID: "a"},
|
||||||
|
}},
|
||||||
|
id: "a",
|
||||||
|
want: []error{ErrInvalidProfile},
|
||||||
|
contains: []string{"a -> b -> c -> a"},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "nil result",
|
||||||
|
repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"leaf": nil,
|
||||||
|
}},
|
||||||
|
id: "leaf",
|
||||||
|
want: []error{ErrInvalidProfile},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "incomplete resolved profile",
|
||||||
|
repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"leaf": {ID: "leaf", BaseProfileID: "base"},
|
||||||
|
"base": {ID: "base", Model: "model"},
|
||||||
|
}},
|
||||||
|
id: "leaf",
|
||||||
|
want: []error{ErrInvalidProfile},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "base source error is retained",
|
||||||
|
repo: &resolvingTestRepository{
|
||||||
|
profiles: map[string]*domain.ExecutionProfile{"leaf": {ID: "leaf", BaseProfileID: "base"}},
|
||||||
|
errors: map[string]error{"base": sourceErr},
|
||||||
|
},
|
||||||
|
id: "leaf",
|
||||||
|
want: []error{ErrInvalidProfile, sourceErr},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
_, err := NewResolvingRepository(tc.repo).GetProfile(context.Background(), tc.id)
|
||||||
|
for _, want := range tc.want {
|
||||||
|
if !errors.Is(err, want) {
|
||||||
|
t.Fatalf("error = %v, want %v", err, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if tc.wantNot != nil && errors.Is(err, tc.wantNot) {
|
||||||
|
t.Fatalf("error = %v, must not match %v", err, tc.wantNot)
|
||||||
|
}
|
||||||
|
for _, fragment := range tc.contains {
|
||||||
|
if !strings.Contains(err.Error(), fragment) {
|
||||||
|
t.Fatalf("error = %v, want %q", err, fragment)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvingRepositoryEnforcesChainLength(t *testing.T) {
|
||||||
|
for _, count := range []int{maximumProfileChainLength, maximumProfileChainLength + 1} {
|
||||||
|
t.Run(fmt.Sprintf("%d profiles", count), func(t *testing.T) {
|
||||||
|
profiles := make(map[string]*domain.ExecutionProfile, count)
|
||||||
|
for index := 1; index <= count; index++ {
|
||||||
|
id := fmt.Sprintf("profile-%d", index)
|
||||||
|
definition := &domain.ExecutionProfile{ID: id}
|
||||||
|
if index == count {
|
||||||
|
definition.Endpoint = "https://root.example/v1"
|
||||||
|
definition.Model = "model"
|
||||||
|
} else {
|
||||||
|
definition.BaseProfileID = fmt.Sprintf("profile-%d", index+1)
|
||||||
|
}
|
||||||
|
profiles[id] = definition
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := NewResolvingRepository(&resolvingTestRepository{profiles: profiles}).GetProfile(context.Background(), "profile-1")
|
||||||
|
if count == maximumProfileChainLength {
|
||||||
|
if err != nil || got == nil {
|
||||||
|
t.Fatalf("profile = %+v, error = %v, want accepted chain", got, err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !errors.Is(err, ErrInvalidProfile) {
|
||||||
|
t.Fatalf("error = %v, want ErrInvalidProfile", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvingRepositoryIsFreshAndCancellationAware(t *testing.T) {
|
||||||
|
repo := &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"leaf": {ID: "leaf", BaseProfileID: "base"},
|
||||||
|
"base": {ID: "base", Endpoint: "https://base.example/v1", Model: "first", ExtraParams: map[string]any{"nested": map[string]any{"value": "first"}}},
|
||||||
|
}}
|
||||||
|
resolver := NewResolvingRepository(repo)
|
||||||
|
|
||||||
|
first, err := resolver.GetProfile(context.Background(), "leaf")
|
||||||
|
if err != nil || first.Model != "first" {
|
||||||
|
t.Fatalf("first result=(%+v, %v)", first, err)
|
||||||
|
}
|
||||||
|
repo.set("base", &domain.ExecutionProfile{ID: "base", Endpoint: "https://base.example/v1", Model: "second", ExtraParams: map[string]any{"nested": map[string]any{"value": "second"}}})
|
||||||
|
second, err := resolver.GetProfile(context.Background(), "leaf")
|
||||||
|
if err != nil || second.Model != "second" {
|
||||||
|
t.Fatalf("second result=(%+v, %v)", second, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
canceled, cancel := context.WithCancel(context.Background())
|
||||||
|
cancel()
|
||||||
|
if _, err := resolver.GetProfile(canceled, "leaf"); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("canceled lookup error = %v", err)
|
||||||
|
}
|
||||||
|
if got := repo.callCount("leaf"); got != 2 {
|
||||||
|
t.Fatalf("calls after canceled lookup = %d, want 2", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
duringTraversal, cancelDuringTraversal := context.WithCancel(context.Background())
|
||||||
|
repo.afterGet = func(id string) {
|
||||||
|
if id == "leaf" {
|
||||||
|
cancelDuringTraversal()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := resolver.GetProfile(duringTraversal, "leaf"); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("during traversal error = %v", err)
|
||||||
|
}
|
||||||
|
if got := repo.callCount("base"); got != 2 {
|
||||||
|
t.Fatalf("base calls after cancellation = %d, want 2", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
terminalLookup, cancelTerminalLookup := context.WithCancel(context.Background())
|
||||||
|
repo.afterGet = func(id string) {
|
||||||
|
if id == "base" {
|
||||||
|
cancelTerminalLookup()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := resolver.GetProfile(terminalLookup, "leaf"); !errors.Is(err, context.Canceled) {
|
||||||
|
t.Fatalf("terminal lookup cancellation error = %v", err)
|
||||||
|
}
|
||||||
|
if got := repo.callCount("base"); got != 3 {
|
||||||
|
t.Fatalf("base calls after terminal cancellation = %d, want 3", got)
|
||||||
|
}
|
||||||
|
|
||||||
|
repo.afterGet = nil
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
errors := make(chan error, 8)
|
||||||
|
for index := 0; index < cap(errors); index++ {
|
||||||
|
wg.Add(1)
|
||||||
|
go func() {
|
||||||
|
defer wg.Done()
|
||||||
|
resolved, err := resolver.GetProfile(context.Background(), "leaf")
|
||||||
|
if err != nil {
|
||||||
|
errors <- err
|
||||||
|
return
|
||||||
|
}
|
||||||
|
resolved.ExtraParams["nested"].(map[string]any)["value"] = "mutated"
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
wg.Wait()
|
||||||
|
close(errors)
|
||||||
|
for err := range errors {
|
||||||
|
t.Errorf("concurrent resolution: %v", err)
|
||||||
|
}
|
||||||
|
latest, err := resolver.GetProfile(context.Background(), "leaf")
|
||||||
|
if err != nil || latest.ExtraParams["nested"].(map[string]any)["value"] != "second" {
|
||||||
|
t.Fatalf("latest result=(%+v, %v)", latest, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type resolvingTestRepository struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
profiles map[string]*domain.ExecutionProfile
|
||||||
|
errors map[string]error
|
||||||
|
calls map[string]int
|
||||||
|
afterGet func(string)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *resolvingTestRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
r.mu.Lock()
|
||||||
|
if r.calls == nil {
|
||||||
|
r.calls = make(map[string]int)
|
||||||
|
}
|
||||||
|
r.calls[id]++
|
||||||
|
err := r.errors[id]
|
||||||
|
profile := r.profiles[id]
|
||||||
|
afterGet := r.afterGet
|
||||||
|
r.mu.Unlock()
|
||||||
|
if afterGet != nil {
|
||||||
|
afterGet(id)
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if profile == nil {
|
||||||
|
if _, exists := r.profiles[id]; exists {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return nil, ErrProfileNotFound
|
||||||
|
}
|
||||||
|
copy := *profile
|
||||||
|
return ©, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *resolvingTestRepository) set(id string, profile *domain.ExecutionProfile) {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
r.profiles[id] = profile
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *resolvingTestRepository) callCount(id string) int {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
return r.calls[id]
|
||||||
|
}
|
||||||
|
|
||||||
|
func profileTestFS(files map[string]string) fs.FS {
|
||||||
|
fsys := make(fstest.MapFS, len(files))
|
||||||
|
for name, content := range files {
|
||||||
|
fsys[name] = profileMapFile(content)
|
||||||
|
}
|
||||||
|
return fsys
|
||||||
|
}
|
||||||
@@ -892,6 +892,38 @@ output:
|
|||||||
format: text
|
format: text
|
||||||
validation_mode: none
|
validation_mode: none
|
||||||
repair_attempts: -1
|
repair_attempts: -1
|
||||||
|
`,
|
||||||
|
wantErr: true,
|
||||||
|
wantDiagnostic: "repair_attempts",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "repair attempts above maximum",
|
||||||
|
definition: `
|
||||||
|
id: normalization-rule
|
||||||
|
version: "1"
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: test
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: basic
|
||||||
|
repair_attempts: 4
|
||||||
|
`,
|
||||||
|
wantErr: true,
|
||||||
|
wantDiagnostic: "repair_attempts",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "none validation with repair attempts",
|
||||||
|
definition: `
|
||||||
|
id: normalization-rule
|
||||||
|
version: "1"
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: test
|
||||||
|
output:
|
||||||
|
format: text
|
||||||
|
validation_mode: none
|
||||||
|
repair_attempts: 1
|
||||||
`,
|
`,
|
||||||
wantErr: true,
|
wantErr: true,
|
||||||
wantDiagnostic: "repair_attempts",
|
wantDiagnostic: "repair_attempts",
|
||||||
|
|||||||
@@ -106,6 +106,8 @@ func TestRunnerPreparationRejectsInvalidOutputContractsBeforeCompletion(t *testi
|
|||||||
{name: "unsupported format", override: domain.OutputContract{Format: "binary", ValidationMode: domain.ValidationNone}},
|
{name: "unsupported format", override: domain.OutputContract{Format: "binary", ValidationMode: domain.ValidationNone}},
|
||||||
{name: "empty validation mode", override: domain.OutputContract{Format: domain.FormatText}},
|
{name: "empty validation mode", override: domain.OutputContract{Format: domain.FormatText}},
|
||||||
{name: "negative repair attempts", override: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationNone, RepairAttempts: -1}},
|
{name: "negative repair attempts", override: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationNone, RepairAttempts: -1}},
|
||||||
|
{name: "repair attempts above maximum", override: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationBasic, RepairAttempts: 4}},
|
||||||
|
{name: "none validation with repair attempts", override: domain.OutputContract{Format: domain.FormatText, ValidationMode: domain.ValidationNone, RepairAttempts: 1}},
|
||||||
{name: "json schema without path", override: domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema}},
|
{name: "json schema without path", override: domain.OutputContract{Format: domain.FormatJSON, ValidationMode: domain.ValidationJSONSchema}},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -3,11 +3,12 @@ package usecase
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"os"
|
"fmt"
|
||||||
"reflect"
|
"reflect"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/validate"
|
"gitea.maximumdirect.net/eric/promptkit/internal/validate"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -241,49 +242,76 @@ func excessivelyDeepPreparedJSONValue() any {
|
|||||||
return value
|
return value
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerRunPreparedRechecksEnvironmentCredentialBeforeAdmission(t *testing.T) {
|
func TestRunnerRunPreparedCredentialAvailabilityBeforeAdmission(t *testing.T) {
|
||||||
const environmentName = "PROMPTKIT_PREPARED_EXECUTION_TEST_KEY"
|
const environmentName = "PROMPTKIT_PREPARED_EXECUTION_TEST_KEY"
|
||||||
t.Setenv(environmentName, "available-during-preparation")
|
tests := []struct {
|
||||||
|
name string
|
||||||
profile := defaultExecutionProfile()
|
apiKeyRequired bool
|
||||||
profile.APIKeyEnv = environmentName
|
profileEnv bool
|
||||||
validator := &recordingValidationPreparer{plan: &recordingPreparedValidation{}}
|
overrideEnv bool
|
||||||
admitter := &fakeRunAdmitter{}
|
wantFailure bool
|
||||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "unexpected"}}
|
}{
|
||||||
runner := NewRunner(
|
{name: "optional environment becomes unavailable", profileEnv: true},
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
{
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
|
name: "required request environment becomes unavailable",
|
||||||
nil,
|
apiKeyRequired: true,
|
||||||
defaultArtifactReader(),
|
overrideEnv: true,
|
||||||
defaultRenderer(),
|
wantFailure: true,
|
||||||
llmClient,
|
},
|
||||||
validator,
|
|
||||||
admitter,
|
|
||||||
)
|
|
||||||
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
|
||||||
PromptID: "p",
|
|
||||||
ProfileID: "exec",
|
|
||||||
Inputs: singleInputRef(),
|
|
||||||
})
|
|
||||||
if err != nil {
|
|
||||||
t.Fatalf("prepare execution: %v", err)
|
|
||||||
}
|
|
||||||
if err := os.Unsetenv(environmentName); err != nil {
|
|
||||||
t.Fatalf("unset credential environment: %v", err)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
result, err := runner.RunPrepared(context.Background(), prepared)
|
for _, tc := range tests {
|
||||||
if result != nil {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
t.Fatalf("credential failure returned partial result: %+v", result)
|
t.Setenv(environmentName, "available-during-preparation")
|
||||||
}
|
|
||||||
if !errors.Is(err, ErrInvalidRequest) || !errors.Is(err, ErrAPIKeyEnvMissing) {
|
profile := defaultExecutionProfile()
|
||||||
t.Fatalf("credential error identities are missing: %v", err)
|
profile.APIKeyRequired = tc.apiKeyRequired
|
||||||
}
|
if tc.profileEnv {
|
||||||
if len(admitter.backendIDs) != 0 || llmClient.calls != 0 {
|
profile.APIKeyEnv = environmentName
|
||||||
t.Fatalf("credential failure reached admission or generation: admission=%v generation=%d", admitter.backendIDs, llmClient.calls)
|
}
|
||||||
}
|
validator := &recordingValidationPreparer{plan: &recordingPreparedValidation{}}
|
||||||
if _, err := runner.RunPrepared(context.Background(), prepared); !errors.Is(err, ErrInvalidRequest) {
|
admitter := &fakeRunAdmitter{}
|
||||||
t.Fatalf("credential failure did not consume execution: %v", err)
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||||
|
runner := NewRunner(
|
||||||
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
|
||||||
|
nil,
|
||||||
|
defaultArtifactReader(),
|
||||||
|
defaultRenderer(),
|
||||||
|
llmClient,
|
||||||
|
validator,
|
||||||
|
admitter,
|
||||||
|
)
|
||||||
|
request := domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()}
|
||||||
|
if tc.overrideEnv {
|
||||||
|
request.Execution = &domain.ExecutionTargetOverride{APIKeyEnv: environmentName}
|
||||||
|
}
|
||||||
|
prepared, err := runner.PrepareExecution(context.Background(), request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution: %v", err)
|
||||||
|
}
|
||||||
|
t.Setenv(environmentName, "")
|
||||||
|
|
||||||
|
result, err := runner.RunPrepared(context.Background(), prepared)
|
||||||
|
if tc.wantFailure {
|
||||||
|
if result != nil || !errors.Is(err, ErrInvalidRequest) || !errors.Is(err, ErrAPIKeyEnvMissing) {
|
||||||
|
t.Fatalf("required credential result = (%+v, %v)", result, err)
|
||||||
|
}
|
||||||
|
if len(admitter.backendIDs) != 0 || llmClient.calls != 0 {
|
||||||
|
t.Fatalf("required credential reached admission or generation: admission=%v generation=%d", admitter.backendIDs, llmClient.calls)
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
if result == nil || err != nil {
|
||||||
|
t.Fatalf("optional credential result = (%+v, %v), want success", result, err)
|
||||||
|
}
|
||||||
|
if len(admitter.backendIDs) != 1 || llmClient.calls != 1 {
|
||||||
|
t.Fatalf("optional credential admission=%v generation=%d, want one each", admitter.backendIDs, llmClient.calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, err := runner.RunPrepared(context.Background(), prepared); !errors.Is(err, ErrInvalidRequest) {
|
||||||
|
t.Fatalf("execution outcome did not consume handle: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -359,19 +387,20 @@ func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *test
|
|||||||
admitter := &fakeRunAdmitter{}
|
admitter := &fakeRunAdmitter{}
|
||||||
reader := defaultArtifactReader()
|
reader := defaultArtifactReader()
|
||||||
renderer := defaultRenderer()
|
renderer := defaultRenderer()
|
||||||
|
client := &sequenceLLM{responses: []*domain.GenerateResponse{{
|
||||||
|
Content: `{"broken":true}`,
|
||||||
|
Usage: domain.TokenUsage{
|
||||||
|
PromptTokens: 13, CompletionTokens: 17, TotalTokens: 19,
|
||||||
|
CachedTokens: 23, CacheWriteTokens: 29,
|
||||||
|
},
|
||||||
|
}}}
|
||||||
runner := NewRunnerWithRepairer(
|
runner := NewRunnerWithRepairer(
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
|
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
nil,
|
nil,
|
||||||
reader,
|
reader,
|
||||||
renderer,
|
renderer,
|
||||||
&fakeLLM{resp: &domain.GenerateResponse{
|
client,
|
||||||
Content: `{"broken":true}`,
|
|
||||||
Usage: domain.TokenUsage{
|
|
||||||
PromptTokens: 13, CompletionTokens: 17, TotalTokens: 19,
|
|
||||||
CachedTokens: 23, CacheWriteTokens: 29,
|
|
||||||
},
|
|
||||||
}},
|
|
||||||
validator,
|
validator,
|
||||||
repairer,
|
repairer,
|
||||||
admitter,
|
admitter,
|
||||||
@@ -395,6 +424,10 @@ func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *test
|
|||||||
if validator.directValidateCalls != 0 || repairer.calls != 1 {
|
if validator.directValidateCalls != 0 || repairer.calls != 1 {
|
||||||
t.Fatalf("validation/repair calls=(direct=%d repair=%d), want (0, 1)", validator.directValidateCalls, repairer.calls)
|
t.Fatalf("validation/repair calls=(direct=%d repair=%d), want (0, 1)", validator.directValidateCalls, repairer.calls)
|
||||||
}
|
}
|
||||||
|
if len(client.requests) != 1 || len(repairer.reqs) != 1 ||
|
||||||
|
!reflect.DeepEqual(repairer.reqs[0].OriginalMessages, client.requests[0].Prompt.Messages) {
|
||||||
|
t.Fatalf("initial and repair messages = (%#v, %#v)", client.requests, repairer.reqs)
|
||||||
|
}
|
||||||
if result.Validation.Status != domain.ValidationPassed || result.Validation.RepairAttempts != 1 {
|
if result.Validation.Status != domain.ValidationPassed || result.Validation.RepairAttempts != 1 {
|
||||||
t.Fatalf("unexpected repaired validation result: %+v", result.Validation)
|
t.Fatalf("unexpected repaired validation result: %+v", result.Validation)
|
||||||
}
|
}
|
||||||
@@ -417,6 +450,7 @@ func TestRunnerRunPreparedReleasesAdmissionAcrossExecutionErrors(t *testing.T) {
|
|||||||
generationFailure := errors.New("generation failed")
|
generationFailure := errors.New("generation failed")
|
||||||
validationFailure := errors.New("validation failed")
|
validationFailure := errors.New("validation failed")
|
||||||
repairFailure := errors.New("repair failed")
|
repairFailure := errors.New("repair failed")
|
||||||
|
repairInvalidRequest := fmt.Errorf("repair request: %w", llm.ErrInvalidRequest)
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
@@ -424,6 +458,7 @@ func TestRunnerRunPreparedReleasesAdmissionAcrossExecutionErrors(t *testing.T) {
|
|||||||
validation *recordingPreparedValidation
|
validation *recordingPreparedValidation
|
||||||
repairer *fakeRepairer
|
repairer *fakeRepairer
|
||||||
wantError error
|
wantError error
|
||||||
|
wantSource error
|
||||||
}{
|
}{
|
||||||
{
|
{
|
||||||
name: "generation failure",
|
name: "generation failure",
|
||||||
@@ -448,8 +483,51 @@ func TestRunnerRunPreparedReleasesAdmissionAcrossExecutionErrors(t *testing.T) {
|
|||||||
IsValid: false,
|
IsValid: false,
|
||||||
}},
|
}},
|
||||||
},
|
},
|
||||||
repairer: &fakeRepairer{err: repairFailure},
|
repairer: &fakeRepairer{err: repairFailure},
|
||||||
wantError: ErrValidation,
|
wantError: ErrLLMGenerate,
|
||||||
|
wantSource: repairFailure,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "repair invalid request",
|
||||||
|
validation: &recordingPreparedValidation{
|
||||||
|
results: []domain.ValidationResult{{
|
||||||
|
Status: domain.ValidationFailed,
|
||||||
|
Mode: domain.ValidationJSON,
|
||||||
|
Errors: []string{"invalid"},
|
||||||
|
IsValid: false,
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
repairer: &fakeRepairer{err: repairInvalidRequest},
|
||||||
|
wantError: ErrInvalidRequest,
|
||||||
|
wantSource: repairInvalidRequest,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "repair cancellation",
|
||||||
|
validation: &recordingPreparedValidation{
|
||||||
|
results: []domain.ValidationResult{{
|
||||||
|
Status: domain.ValidationFailed,
|
||||||
|
Mode: domain.ValidationJSON,
|
||||||
|
Errors: []string{"invalid"},
|
||||||
|
IsValid: false,
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
repairer: &fakeRepairer{err: context.Canceled},
|
||||||
|
wantError: ErrLLMGenerate,
|
||||||
|
wantSource: context.Canceled,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "repair deadline",
|
||||||
|
validation: &recordingPreparedValidation{
|
||||||
|
results: []domain.ValidationResult{{
|
||||||
|
Status: domain.ValidationFailed,
|
||||||
|
Mode: domain.ValidationJSON,
|
||||||
|
Errors: []string{"invalid"},
|
||||||
|
IsValid: false,
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
repairer: &fakeRepairer{err: context.DeadlineExceeded},
|
||||||
|
wantError: ErrLLMGenerate,
|
||||||
|
wantSource: context.DeadlineExceeded,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -488,6 +566,9 @@ func TestRunnerRunPreparedReleasesAdmissionAcrossExecutionErrors(t *testing.T) {
|
|||||||
if result != nil || !errors.Is(err, test.wantError) {
|
if result != nil || !errors.Is(err, test.wantError) {
|
||||||
t.Fatalf("run prepared=(%+v, %v), want %v", result, err, test.wantError)
|
t.Fatalf("run prepared=(%+v, %v), want %v", result, err, test.wantError)
|
||||||
}
|
}
|
||||||
|
if test.wantSource != nil && !errors.Is(err, test.wantSource) {
|
||||||
|
t.Fatalf("run prepared error = %v, want source %v", err, test.wantSource)
|
||||||
|
}
|
||||||
if len(admitter.backendIDs) != 1 || admitter.releaseCalls != 1 {
|
if len(admitter.backendIDs) != 1 || admitter.releaseCalls != 1 {
|
||||||
t.Fatalf(
|
t.Fatalf(
|
||||||
"admission calls=%#v releases=%d, want one each",
|
"admission calls=%#v releases=%d, want one each",
|
||||||
|
|||||||
@@ -2,19 +2,29 @@ package usecase
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
|
"unicode/utf8"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
maxRepairDiagnosticBytes = 64 * 1024
|
||||||
|
omittedRepairDiagnostics = "additional validation diagnostics were omitted"
|
||||||
|
)
|
||||||
|
|
||||||
|
// OutputRepairer generates a corrected candidate after validation fails.
|
||||||
type OutputRepairer interface {
|
type OutputRepairer interface {
|
||||||
Repair(ctx context.Context, req RepairRequest) (*domain.GenerateResponse, error)
|
Repair(ctx context.Context, req RepairRequest) (*domain.GenerateResponse, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RepairRequest contains the immutable execution state needed for one correction.
|
||||||
type RepairRequest struct {
|
type RepairRequest struct {
|
||||||
|
OriginalMessages []domain.RenderedMessage
|
||||||
PreviousOutput string
|
PreviousOutput string
|
||||||
ValidationErrors []string
|
ValidationErrors []string
|
||||||
SessionID string
|
SessionID string
|
||||||
@@ -30,6 +40,7 @@ type defaultOutputRepairer struct {
|
|||||||
llm llm.Client
|
llm llm.Client
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NewDefaultOutputRepairer constructs the standard internal output repairer.
|
||||||
func NewDefaultOutputRepairer(llmClient llm.Client) OutputRepairer {
|
func NewDefaultOutputRepairer(llmClient llm.Client) OutputRepairer {
|
||||||
return &defaultOutputRepairer{llm: llmClient}
|
return &defaultOutputRepairer{llm: llmClient}
|
||||||
}
|
}
|
||||||
@@ -39,33 +50,42 @@ func (r *defaultOutputRepairer) Repair(ctx context.Context, req RepairRequest) (
|
|||||||
return nil, errors.New("llm client is required for repair")
|
return nil, errors.New("llm client is required for repair")
|
||||||
}
|
}
|
||||||
|
|
||||||
errs := "(none provided)"
|
guidance, err := repairGuidance(req.Mode)
|
||||||
if len(req.ValidationErrors) > 0 {
|
if err != nil {
|
||||||
errs = strings.Join(req.ValidationErrors, "\n")
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
prompt := domain.RenderedPrompt{
|
messages := make([]domain.RenderedMessage, len(req.OriginalMessages), len(req.OriginalMessages)+2)
|
||||||
Messages: []domain.RenderedMessage{
|
copy(messages, req.OriginalMessages)
|
||||||
{
|
if strings.TrimSpace(req.PreviousOutput) != "" {
|
||||||
Role: "system",
|
messages = append(messages, domain.RenderedMessage{
|
||||||
Content: "You repair invalid JSON output. Return only corrected JSON. Do not include explanations or markdown code fences.",
|
Role: "assistant",
|
||||||
},
|
Content: req.PreviousOutput,
|
||||||
{
|
})
|
||||||
Role: "user",
|
|
||||||
Content: fmt.Sprintf(
|
|
||||||
"Repair attempt %d of %d for validation mode %s.\n\nValidation errors:\n%s\n\nPrevious output:\n%s\n\nReturn only corrected JSON.",
|
|
||||||
req.Attempt,
|
|
||||||
req.MaxAttempts,
|
|
||||||
req.Mode,
|
|
||||||
errs,
|
|
||||||
req.PreviousOutput,
|
|
||||||
),
|
|
||||||
},
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
previousResponse := "The previous response was empty."
|
||||||
|
if strings.TrimSpace(req.PreviousOutput) != "" {
|
||||||
|
previousResponse = "The previous response is included immediately before this instruction."
|
||||||
|
}
|
||||||
|
messages = append(messages, domain.RenderedMessage{
|
||||||
|
Role: "user",
|
||||||
|
Content: fmt.Sprintf(
|
||||||
|
"Repair attempt %d of %d for validation mode %s.\n"+
|
||||||
|
"Preserve valid values and change only what is necessary.\n"+
|
||||||
|
"%s\n%s\n"+
|
||||||
|
"Validation diagnostics (data):\n%s",
|
||||||
|
req.Attempt,
|
||||||
|
req.MaxAttempts,
|
||||||
|
req.Mode,
|
||||||
|
previousResponse,
|
||||||
|
guidance,
|
||||||
|
formatRepairDiagnostics(req.ValidationErrors),
|
||||||
|
),
|
||||||
|
})
|
||||||
|
|
||||||
resp, err := r.llm.Generate(ctx, newGenerationRequest(
|
resp, err := r.llm.Generate(ctx, newGenerationRequest(
|
||||||
prompt,
|
domain.RenderedPrompt{Messages: messages},
|
||||||
req.SessionID,
|
req.SessionID,
|
||||||
req.Target,
|
req.Target,
|
||||||
req.TargetPresence,
|
req.TargetPresence,
|
||||||
@@ -80,3 +100,90 @@ func (r *defaultOutputRepairer) Repair(ctx context.Context, req RepairRequest) (
|
|||||||
|
|
||||||
return resp, nil
|
return resp, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func repairGuidance(mode domain.ValidationMode) (string, error) {
|
||||||
|
switch mode {
|
||||||
|
case domain.ValidationBasic:
|
||||||
|
return "Return a nonempty response satisfying the original request.", nil
|
||||||
|
case domain.ValidationJSON, domain.ValidationJSONSchema:
|
||||||
|
return "Return only corrected JSON, with no explanation or Markdown fences.", nil
|
||||||
|
default:
|
||||||
|
return "", fmt.Errorf("unsupported validation mode for repair: %q", mode)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func formatRepairDiagnostics(errors []string) string {
|
||||||
|
diagnostics := make([]string, len(errors))
|
||||||
|
for index, diagnostic := range errors {
|
||||||
|
diagnostics[index] = strings.ToValidUTF8(diagnostic, "\uFFFD")
|
||||||
|
}
|
||||||
|
|
||||||
|
complete, _ := json.Marshal(diagnostics)
|
||||||
|
if len(complete) <= maxRepairDiagnosticBytes {
|
||||||
|
return string(complete)
|
||||||
|
}
|
||||||
|
|
||||||
|
omission, _ := json.Marshal(omittedRepairDiagnostics)
|
||||||
|
encoded := make([]byte, 0, maxRepairDiagnosticBytes)
|
||||||
|
encoded = append(encoded, '[')
|
||||||
|
for _, diagnostic := range diagnostics {
|
||||||
|
entry, _ := json.Marshal(diagnostic)
|
||||||
|
separator := 0
|
||||||
|
if len(encoded) > 1 {
|
||||||
|
separator = 1
|
||||||
|
}
|
||||||
|
available := maxRepairDiagnosticBytes - len(encoded) - separator - 1 - len(omission) - 1
|
||||||
|
if len(entry) <= available {
|
||||||
|
if separator != 0 {
|
||||||
|
encoded = append(encoded, ',')
|
||||||
|
}
|
||||||
|
encoded = append(encoded, entry...)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if available < len(`""`) {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if separator != 0 {
|
||||||
|
encoded = append(encoded, ',')
|
||||||
|
}
|
||||||
|
encoded = append(encoded, truncateDiagnosticJSONValue(diagnostic, available)...)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if len(encoded) > 1 {
|
||||||
|
encoded = append(encoded, ',')
|
||||||
|
}
|
||||||
|
encoded = append(encoded, omission...)
|
||||||
|
encoded = append(encoded, ']')
|
||||||
|
return string(encoded)
|
||||||
|
}
|
||||||
|
|
||||||
|
func truncateDiagnosticJSONValue(value string, maxBytes int) []byte {
|
||||||
|
if maxBytes < len(`""`) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
boundaries := []int{0}
|
||||||
|
for end := 0; end < len(value); {
|
||||||
|
_, size := utf8.DecodeRuneInString(value[end:])
|
||||||
|
end += size
|
||||||
|
if end+len(`""`) > maxBytes {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
boundaries = append(boundaries, end)
|
||||||
|
}
|
||||||
|
|
||||||
|
low, high := 0, len(boundaries)-1
|
||||||
|
best := []byte(`""`)
|
||||||
|
for low <= high {
|
||||||
|
mid := low + (high-low)/2
|
||||||
|
candidate, _ := json.Marshal(value[:boundaries[mid]])
|
||||||
|
if len(candidate) <= maxBytes {
|
||||||
|
best = candidate
|
||||||
|
low = mid + 1
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
high = mid - 1
|
||||||
|
}
|
||||||
|
return best
|
||||||
|
}
|
||||||
|
|||||||
349
internal/usecase/repairer_test.go
Normal file
349
internal/usecase/repairer_test.go
Normal file
@@ -0,0 +1,349 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"unicode/utf8"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
type recordingRepairClient struct {
|
||||||
|
requests []domain.GenerateRequest
|
||||||
|
response *domain.GenerateResponse
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *recordingRepairClient) Generate(_ context.Context, req domain.GenerateRequest) (*domain.GenerateResponse, error) {
|
||||||
|
c.requests = append(c.requests, req)
|
||||||
|
return c.response, c.err
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDefaultOutputRepairerBuildsFullContextRequest(t *testing.T) {
|
||||||
|
client := &recordingRepairClient{response: &domain.GenerateResponse{Content: "corrected"}}
|
||||||
|
repairer := NewDefaultOutputRepairer(client)
|
||||||
|
original := []domain.RenderedMessage{
|
||||||
|
{Role: "system", Content: "Follow the task.", CacheControl: &domain.CacheControl{Type: domain.CacheControlEphemeral, TTL: "1h"}},
|
||||||
|
{Role: "user", Content: "Summarize the report."},
|
||||||
|
}
|
||||||
|
before := append([]domain.RenderedMessage(nil), original...)
|
||||||
|
previous := strings.Repeat("candidate ", 12_000)
|
||||||
|
target := domain.ExecutionTarget{BackendID: "backend", Endpoint: "https://provider.example/v1", Model: "model"}
|
||||||
|
presence := domain.ExecutionTargetPresence{Temperature: true, TopP: true}
|
||||||
|
structured := &domain.StructuredOutputSpec{}
|
||||||
|
|
||||||
|
response, err := repairer.Repair(context.Background(), RepairRequest{
|
||||||
|
OriginalMessages: original,
|
||||||
|
PreviousOutput: previous,
|
||||||
|
ValidationErrors: []string{"invalid JSON"},
|
||||||
|
SessionID: "session",
|
||||||
|
Target: target,
|
||||||
|
TargetPresence: presence,
|
||||||
|
StructuredOutput: structured,
|
||||||
|
Attempt: 1,
|
||||||
|
MaxAttempts: 3,
|
||||||
|
Mode: domain.ValidationJSONSchema,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("repair: %v", err)
|
||||||
|
}
|
||||||
|
if response == nil || response.Content != "corrected" {
|
||||||
|
t.Fatalf("response = %+v", response)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(original, before) {
|
||||||
|
t.Fatalf("original messages changed: got %#v, want %#v", original, before)
|
||||||
|
}
|
||||||
|
if len(client.requests) != 1 {
|
||||||
|
t.Fatalf("generation requests = %d, want 1", len(client.requests))
|
||||||
|
}
|
||||||
|
|
||||||
|
request := client.requests[0]
|
||||||
|
if request.Prompt.SessionID != "session" || !reflect.DeepEqual(request.Target, target) || request.TargetPresence != presence || request.StructuredOutput != structured {
|
||||||
|
t.Fatalf("generation request fields = %+v", request)
|
||||||
|
}
|
||||||
|
if len(request.Prompt.Messages) != len(original)+2 {
|
||||||
|
t.Fatalf("message count = %d, want %d", len(request.Prompt.Messages), len(original)+2)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(request.Prompt.Messages[:len(original)], original) {
|
||||||
|
t.Fatalf("original messages = %#v, want %#v", request.Prompt.Messages[:len(original)], original)
|
||||||
|
}
|
||||||
|
assistant := request.Prompt.Messages[len(original)]
|
||||||
|
if assistant.Role != "assistant" || assistant.Content != previous {
|
||||||
|
t.Fatalf("assistant candidate = %+v", assistant)
|
||||||
|
}
|
||||||
|
correction := request.Prompt.Messages[len(original)+1]
|
||||||
|
if correction.Role != "user" || !strings.Contains(correction.Content, "Repair attempt 1 of 3") ||
|
||||||
|
!strings.Contains(correction.Content, "Preserve valid values") ||
|
||||||
|
!strings.Contains(correction.Content, "Return only corrected JSON") {
|
||||||
|
t.Fatalf("correction message = %q", correction.Content)
|
||||||
|
}
|
||||||
|
if diagnostics := repairDiagnosticsFromMessage(t, correction.Content); !reflect.DeepEqual(diagnostics, []string{"invalid JSON"}) {
|
||||||
|
t.Fatalf("diagnostics = %#v", diagnostics)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDefaultOutputRepairerDoesNotAccumulateCandidates(t *testing.T) {
|
||||||
|
client := &recordingRepairClient{response: &domain.GenerateResponse{Content: "corrected"}}
|
||||||
|
repairer := NewDefaultOutputRepairer(client)
|
||||||
|
backing := make([]domain.RenderedMessage, 1, 4)
|
||||||
|
backing[0] = domain.RenderedMessage{Role: "user", Content: "Original task"}
|
||||||
|
before := append([]domain.RenderedMessage(nil), backing...)
|
||||||
|
|
||||||
|
for _, candidate := range []string{"first invalid", "second invalid"} {
|
||||||
|
_, err := repairer.Repair(context.Background(), RepairRequest{
|
||||||
|
OriginalMessages: backing,
|
||||||
|
PreviousOutput: candidate,
|
||||||
|
Attempt: 1,
|
||||||
|
MaxAttempts: 3,
|
||||||
|
Mode: domain.ValidationJSON,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("repair %q: %v", candidate, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(backing, before) {
|
||||||
|
t.Fatalf("caller messages changed: got %#v, want %#v", backing, before)
|
||||||
|
}
|
||||||
|
if len(client.requests) != 2 {
|
||||||
|
t.Fatalf("generation requests = %d, want 2", len(client.requests))
|
||||||
|
}
|
||||||
|
for index, request := range client.requests {
|
||||||
|
messages := request.Prompt.Messages
|
||||||
|
if len(messages) != 3 || messages[0] != backing[0] || messages[1].Role != "assistant" || messages[1].Content != []string{"first invalid", "second invalid"}[index] {
|
||||||
|
t.Fatalf("request %d messages = %#v", index, messages)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDefaultOutputRepairerOmitsEmptyCandidateMessage(t *testing.T) {
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
candidate string
|
||||||
|
}{
|
||||||
|
{name: "empty", candidate: ""},
|
||||||
|
{name: "whitespace", candidate: " \n\t "},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
client := &recordingRepairClient{response: &domain.GenerateResponse{Content: "corrected"}}
|
||||||
|
repairer := NewDefaultOutputRepairer(client)
|
||||||
|
_, err := repairer.Repair(context.Background(), RepairRequest{
|
||||||
|
OriginalMessages: []domain.RenderedMessage{{Role: "user", Content: "Original task"}},
|
||||||
|
PreviousOutput: tc.candidate,
|
||||||
|
Attempt: 1,
|
||||||
|
MaxAttempts: 1,
|
||||||
|
Mode: domain.ValidationBasic,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("repair: %v", err)
|
||||||
|
}
|
||||||
|
messages := client.requests[0].Prompt.Messages
|
||||||
|
if len(messages) != 2 || messages[1].Role != "user" || !strings.Contains(messages[1].Content, "previous response was empty") {
|
||||||
|
t.Fatalf("messages = %#v", messages)
|
||||||
|
}
|
||||||
|
if strings.Contains(messages[1].Content, tc.candidate) && tc.candidate != "" {
|
||||||
|
t.Fatalf("correction message repeated whitespace candidate: %q", messages[1].Content)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDefaultOutputRepairerUsesModeSpecificGuidance(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
mode domain.ValidationMode
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{mode: domain.ValidationBasic, want: "Return a nonempty response"},
|
||||||
|
{mode: domain.ValidationJSON, want: "Return only corrected JSON"},
|
||||||
|
{mode: domain.ValidationJSONSchema, want: "Return only corrected JSON"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(string(tc.mode), func(t *testing.T) {
|
||||||
|
client := &recordingRepairClient{response: &domain.GenerateResponse{Content: "corrected"}}
|
||||||
|
_, err := NewDefaultOutputRepairer(client).Repair(context.Background(), RepairRequest{Mode: tc.mode, Attempt: 1, MaxAttempts: 1})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("repair: %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(client.requests[0].Prompt.Messages[0].Content, tc.want) {
|
||||||
|
t.Fatalf("correction message = %q", client.requests[0].Prompt.Messages[0].Content)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
client := &recordingRepairClient{response: &domain.GenerateResponse{Content: "unexpected"}}
|
||||||
|
_, err := NewDefaultOutputRepairer(client).Repair(context.Background(), RepairRequest{Mode: domain.ValidationNone})
|
||||||
|
if err == nil || !strings.Contains(err.Error(), "unsupported validation mode") {
|
||||||
|
t.Fatalf("unsupported mode error = %v", err)
|
||||||
|
}
|
||||||
|
if len(client.requests) != 0 {
|
||||||
|
t.Fatalf("generation requests = %d, want 0", len(client.requests))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestFormatRepairDiagnosticsBoundsAndPreservesData(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
errors []string
|
||||||
|
wantOmission bool
|
||||||
|
check func(*testing.T, []string)
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "below limit",
|
||||||
|
errors: []string{"first", "second"},
|
||||||
|
check: func(t *testing.T, got []string) {
|
||||||
|
t.Helper()
|
||||||
|
if !reflect.DeepEqual(got, []string{"first", "second"}) {
|
||||||
|
t.Fatalf("diagnostics = %#v", got)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "at limit",
|
||||||
|
errors: []string{strings.Repeat("x", maxRepairDiagnosticBytes-4)},
|
||||||
|
check: func(t *testing.T, got []string) {
|
||||||
|
t.Helper()
|
||||||
|
if len(got) != 1 || len(got[0]) != maxRepairDiagnosticBytes-4 {
|
||||||
|
t.Fatalf("diagnostics lengths = %#v", got)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "multibyte truncation preserves prior entries",
|
||||||
|
errors: []string{"first", strings.Repeat("界", maxRepairDiagnosticBytes)},
|
||||||
|
wantOmission: true,
|
||||||
|
check: func(t *testing.T, got []string) {
|
||||||
|
t.Helper()
|
||||||
|
if len(got) != 3 || got[0] != "first" || !strings.HasPrefix(strings.Repeat("界", maxRepairDiagnosticBytes), got[1]) {
|
||||||
|
t.Fatalf("diagnostics = %#v", got)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "many diagnostics",
|
||||||
|
errors: manyRepairDiagnostics(),
|
||||||
|
wantOmission: true,
|
||||||
|
check: func(t *testing.T, got []string) {
|
||||||
|
t.Helper()
|
||||||
|
if len(got) < 2 || got[0] != manyRepairDiagnostics()[0] {
|
||||||
|
t.Fatalf("diagnostics = %#v", got)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "no room for partial diagnostic",
|
||||||
|
errors: func() []string {
|
||||||
|
omission, _ := json.Marshal(omittedRepairDiagnostics)
|
||||||
|
return []string{
|
||||||
|
strings.Repeat("x", maxRepairDiagnosticBytes-len(omission)-5),
|
||||||
|
strings.Repeat("y", 128),
|
||||||
|
}
|
||||||
|
}(),
|
||||||
|
wantOmission: true,
|
||||||
|
check: func(t *testing.T, got []string) {
|
||||||
|
t.Helper()
|
||||||
|
if len(got) != 2 || got[1] != omittedRepairDiagnostics {
|
||||||
|
t.Fatalf("diagnostics = %#v", got)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid UTF-8",
|
||||||
|
errors: []string{"broken\xffinput"},
|
||||||
|
check: func(t *testing.T, got []string) {
|
||||||
|
t.Helper()
|
||||||
|
if !reflect.DeepEqual(got, []string{"broken\uFFFDinput"}) {
|
||||||
|
t.Fatalf("diagnostics = %#v", got)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "one huge diagnostic",
|
||||||
|
errors: []string{strings.Repeat("x", maxRepairDiagnosticBytes*2)},
|
||||||
|
wantOmission: true,
|
||||||
|
check: func(t *testing.T, got []string) {
|
||||||
|
t.Helper()
|
||||||
|
if len(got) != 2 || !strings.HasPrefix(strings.Repeat("x", maxRepairDiagnosticBytes*2), got[0]) {
|
||||||
|
t.Fatalf("diagnostics = %#v", got)
|
||||||
|
}
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
before := append([]string(nil), tc.errors...)
|
||||||
|
encoded := formatRepairDiagnostics(tc.errors)
|
||||||
|
if len(encoded) > maxRepairDiagnosticBytes || !utf8.ValidString(encoded) {
|
||||||
|
t.Fatalf("encoded diagnostic length/UTF-8 = (%d, %v)", len(encoded), utf8.ValidString(encoded))
|
||||||
|
}
|
||||||
|
var got []string
|
||||||
|
if err := json.Unmarshal([]byte(encoded), &got); err != nil {
|
||||||
|
t.Fatalf("decode diagnostics: %v; encoded=%q", err, encoded)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(tc.errors, before) {
|
||||||
|
t.Fatalf("input diagnostics changed: got %#v, want %#v", tc.errors, before)
|
||||||
|
}
|
||||||
|
if hasOmission := len(got) > 0 && got[len(got)-1] == omittedRepairDiagnostics; hasOmission != tc.wantOmission {
|
||||||
|
t.Fatalf("omission = %v, want %v; diagnostics=%#v", hasOmission, tc.wantOmission, got)
|
||||||
|
}
|
||||||
|
tc.check(t, got)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func manyRepairDiagnostics() []string {
|
||||||
|
diagnostics := make([]string, 1_000)
|
||||||
|
for index := range diagnostics {
|
||||||
|
diagnostics[index] = fmt.Sprintf("diagnostic %04d %s", index, strings.Repeat("x", 128))
|
||||||
|
}
|
||||||
|
return diagnostics
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDefaultOutputRepairerPropagatesGenerationFailures(t *testing.T) {
|
||||||
|
expected := errors.New("generation failed")
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
client *recordingRepairClient
|
||||||
|
want error
|
||||||
|
}{
|
||||||
|
{name: "nil client", want: nil},
|
||||||
|
{name: "generation error", client: &recordingRepairClient{err: expected}, want: expected},
|
||||||
|
{name: "nil response", client: &recordingRepairClient{}, want: nil},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
var repairer OutputRepairer
|
||||||
|
if tc.client == nil {
|
||||||
|
repairer = NewDefaultOutputRepairer(nil)
|
||||||
|
} else {
|
||||||
|
repairer = NewDefaultOutputRepairer(tc.client)
|
||||||
|
}
|
||||||
|
response, err := repairer.Repair(context.Background(), RepairRequest{Mode: domain.ValidationJSON})
|
||||||
|
if response != nil || err == nil {
|
||||||
|
t.Fatalf("response/error = (%+v, %v)", response, err)
|
||||||
|
}
|
||||||
|
if tc.want != nil && !errors.Is(err, tc.want) {
|
||||||
|
t.Fatalf("error = %v, want %v", err, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func repairDiagnosticsFromMessage(t *testing.T, message string) []string {
|
||||||
|
t.Helper()
|
||||||
|
const marker = "Validation diagnostics (data):\n"
|
||||||
|
index := strings.Index(message, marker)
|
||||||
|
if index < 0 {
|
||||||
|
t.Fatalf("missing diagnostics marker in %q", message)
|
||||||
|
}
|
||||||
|
encoded := message[index+len(marker):]
|
||||||
|
var diagnostics []string
|
||||||
|
if err := json.Unmarshal([]byte(encoded), &diagnostics); err != nil {
|
||||||
|
t.Fatalf("decode diagnostics: %v", err)
|
||||||
|
}
|
||||||
|
return diagnostics
|
||||||
|
}
|
||||||
@@ -184,10 +184,10 @@ func (r *Runner) executePreparedRun(
|
|||||||
prepared.StructuredOutput,
|
prepared.StructuredOutput,
|
||||||
))
|
))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
if errors.Is(err, llm.ErrInvalidRequest) {
|
return nil, wrapGenerationError(err)
|
||||||
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
}
|
||||||
}
|
if genResp == nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrLLMGenerate, err)
|
return nil, fmt.Errorf("%w: model returned nil response", ErrLLMGenerate)
|
||||||
}
|
}
|
||||||
usage := genResp.Usage
|
usage := genResp.Usage
|
||||||
|
|
||||||
@@ -203,6 +203,7 @@ func (r *Runner) executePreparedRun(
|
|||||||
attemptsUsed++
|
attemptsUsed++
|
||||||
|
|
||||||
repairResp, repairErr := r.repairer.Repair(ctx, RepairRequest{
|
repairResp, repairErr := r.repairer.Repair(ctx, RepairRequest{
|
||||||
|
OriginalMessages: prepared.Messages,
|
||||||
PreviousOutput: genResp.Content,
|
PreviousOutput: genResp.Content,
|
||||||
ValidationErrors: validationResult.Errors,
|
ValidationErrors: validationResult.Errors,
|
||||||
SessionID: prepared.SessionID,
|
SessionID: prepared.SessionID,
|
||||||
@@ -214,10 +215,10 @@ func (r *Runner) executePreparedRun(
|
|||||||
Mode: prepared.OutputContract.ValidationMode,
|
Mode: prepared.OutputContract.ValidationMode,
|
||||||
})
|
})
|
||||||
if repairErr != nil {
|
if repairErr != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrValidation, repairErr)
|
return nil, wrapGenerationError(repairErr)
|
||||||
}
|
}
|
||||||
if repairResp == nil {
|
if repairResp == nil {
|
||||||
return nil, fmt.Errorf("%w: repairer returned nil response", ErrValidation)
|
return nil, fmt.Errorf("%w: repairer returned nil response", ErrLLMGenerate)
|
||||||
}
|
}
|
||||||
|
|
||||||
genResp = repairResp
|
genResp = repairResp
|
||||||
@@ -257,6 +258,13 @@ func (r *Runner) executePreparedRun(
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func wrapGenerationError(err error) error {
|
||||||
|
if errors.Is(err, llm.ErrInvalidRequest) {
|
||||||
|
return fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("%w: %w", ErrLLMGenerate, err)
|
||||||
|
}
|
||||||
|
|
||||||
func addTokenUsage(total, next domain.TokenUsage) domain.TokenUsage {
|
func addTokenUsage(total, next domain.TokenUsage) domain.TokenUsage {
|
||||||
return domain.TokenUsage{
|
return domain.TokenUsage{
|
||||||
PromptTokens: total.PromptTokens + next.PromptTokens,
|
PromptTokens: total.PromptTokens + next.PromptTokens,
|
||||||
@@ -524,7 +532,12 @@ func (r *Runner) shouldAttemptRepair(contract domain.OutputContract, validationR
|
|||||||
if validationResult.Status != domain.ValidationFailed {
|
if validationResult.Status != domain.ValidationFailed {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
return contract.ValidationMode == domain.ValidationJSON || contract.ValidationMode == domain.ValidationJSONSchema
|
switch contract.ValidationMode {
|
||||||
|
case domain.ValidationBasic, domain.ValidationJSON, domain.ValidationJSONSchema:
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func mergeExecutionTarget(base domain.ExecutionTarget, override domain.ExecutionTarget) domain.ExecutionTarget {
|
func mergeExecutionTarget(base domain.ExecutionTarget, override domain.ExecutionTarget) domain.ExecutionTarget {
|
||||||
@@ -624,12 +637,12 @@ func validateAPIKey(apiKeyEnv string, apiKey string, apiKeyRequired bool) error
|
|||||||
if strings.TrimSpace(apiKey) != "" {
|
if strings.TrimSpace(apiKey) != "" {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
if !apiKeyRequired {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
envName := strings.TrimSpace(apiKeyEnv)
|
envName := strings.TrimSpace(apiKeyEnv)
|
||||||
if envName == "" {
|
if envName == "" {
|
||||||
if apiKeyRequired {
|
return ErrAPIKeyRequired
|
||||||
return ErrAPIKeyRequired
|
|
||||||
}
|
|
||||||
return nil
|
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(os.Getenv(envName)) == "" {
|
if strings.TrimSpace(os.Getenv(envName)) == "" {
|
||||||
return fmt.Errorf("%w: api key environment variable %q is not set", ErrAPIKeyEnvMissing, envName)
|
return fmt.Errorf("%w: api key environment variable %q is not set", ErrAPIKeyEnvMissing, envName)
|
||||||
|
|||||||
@@ -295,8 +295,10 @@ func (c *controlledRepairLLM) Generate(
|
|||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
req domain.GenerateRequest,
|
req domain.GenerateRequest,
|
||||||
) (*domain.GenerateResponse, error) {
|
) (*domain.GenerateResponse, error) {
|
||||||
isRepair := len(req.Prompt.Messages) > 0 &&
|
messages := req.Prompt.Messages
|
||||||
strings.HasPrefix(req.Prompt.Messages[0].Content, "You repair invalid JSON")
|
isRepair := len(messages) >= 2 &&
|
||||||
|
messages[len(messages)-2].Role == "assistant" &&
|
||||||
|
messages[len(messages)-1].Role == "user"
|
||||||
c.mu.Lock()
|
c.mu.Lock()
|
||||||
c.calls++
|
c.calls++
|
||||||
c.active++
|
c.active++
|
||||||
@@ -1787,22 +1789,26 @@ func TestRunnerRunAPIKeyEnvResolvesFromEnvironment(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerRunAPIKeyEnvMissingEnvironmentValueFailsClearly(t *testing.T) {
|
func TestRunnerRunOptionalAPIKeyEnvMissingEnvironmentValueReachesLLM(t *testing.T) {
|
||||||
|
const environmentName = "PROMPTKIT_MISSING_KEY"
|
||||||
|
t.Setenv(environmentName, "")
|
||||||
|
|
||||||
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
||||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_MISSING_KEY"},
|
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: environmentName},
|
||||||
}}
|
}}
|
||||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil, nil)
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||||
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil)
|
||||||
|
|
||||||
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
result, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||||
if !errors.Is(err, ErrInvalidRequest) {
|
if err != nil || result == nil {
|
||||||
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
t.Fatalf("optional credential run = (%+v, %v), want success", result, err)
|
||||||
}
|
}
|
||||||
if !errors.Is(err, ErrAPIKeyEnvMissing) {
|
if llmClient.calls != 1 {
|
||||||
t.Fatalf("expected ErrAPIKeyEnvMissing, got %v", err)
|
t.Fatalf("LLM calls = %d, want 1", llmClient.calls)
|
||||||
}
|
}
|
||||||
if !strings.Contains(err.Error(), "PROMPTKIT_MISSING_KEY") {
|
if llmClient.lastReq.Target.APIKeyEnv != environmentName {
|
||||||
t.Fatalf("expected missing env name in error, got %v", err)
|
t.Fatalf("LLM api_key_env = %q, want %q", llmClient.lastReq.Target.APIKeyEnv, environmentName)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -2139,6 +2145,8 @@ func TestRunnerRepairStateMachine(t *testing.T) {
|
|||||||
TopP: true,
|
TopP: true,
|
||||||
TimeoutSeconds: true,
|
TimeoutSeconds: true,
|
||||||
}
|
}
|
||||||
|
emptyThenValid := responses(2)
|
||||||
|
emptyThenValid[0].Content = " \t "
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
@@ -2161,12 +2169,13 @@ func TestRunnerRepairStateMachine(t *testing.T) {
|
|||||||
wantStatus: domain.ValidationPassed,
|
wantStatus: domain.ValidationPassed,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "basic failure is ineligible despite budget",
|
name: "empty basic output repairs successfully",
|
||||||
mode: domain.ValidationBasic,
|
mode: domain.ValidationBasic,
|
||||||
budget: 3,
|
budget: 3,
|
||||||
validationResults: []domain.ValidationResult{failed(domain.ValidationBasic, "empty output")},
|
validationResults: []domain.ValidationResult{failed(domain.ValidationBasic, "empty output"), passed(domain.ValidationBasic)},
|
||||||
responses: responses(1),
|
responses: emptyThenValid,
|
||||||
wantStatus: domain.ValidationFailed,
|
wantRepairs: 1,
|
||||||
|
wantStatus: domain.ValidationPassed,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
name: "inherited numeric values remain absent",
|
name: "inherited numeric values remain absent",
|
||||||
@@ -2198,7 +2207,7 @@ func TestRunnerRepairStateMachine(t *testing.T) {
|
|||||||
{
|
{
|
||||||
name: "successful repair stops below larger budget",
|
name: "successful repair stops below larger budget",
|
||||||
mode: domain.ValidationJSON,
|
mode: domain.ValidationJSON,
|
||||||
budget: 4,
|
budget: 3,
|
||||||
validationResults: []domain.ValidationResult{
|
validationResults: []domain.ValidationResult{
|
||||||
failed(domain.ValidationJSON, "candidate zero"),
|
failed(domain.ValidationJSON, "candidate zero"),
|
||||||
failed(domain.ValidationJSON, "candidate one"),
|
failed(domain.ValidationJSON, "candidate one"),
|
||||||
@@ -2301,6 +2310,9 @@ func TestRunnerRepairStateMachine(t *testing.T) {
|
|||||||
!reflect.DeepEqual(req.ValidationErrors, tc.validationResults[index].Errors) {
|
!reflect.DeepEqual(req.ValidationErrors, tc.validationResults[index].Errors) {
|
||||||
t.Fatalf("repair request %d prior state = %+v", index, req)
|
t.Fatalf("repair request %d prior state = %+v", index, req)
|
||||||
}
|
}
|
||||||
|
if !reflect.DeepEqual(req.OriginalMessages, initialRequest.Prompt.Messages) {
|
||||||
|
t.Fatalf("repair request %d original messages drifted: %#v", index, req.OriginalMessages)
|
||||||
|
}
|
||||||
if req.TargetPresence != tc.wantPresence || !reflect.DeepEqual(req.Target, initialRequest.Target) ||
|
if req.TargetPresence != tc.wantPresence || !reflect.DeepEqual(req.Target, initialRequest.Target) ||
|
||||||
req.SessionID != initialRequest.Prompt.SessionID ||
|
req.SessionID != initialRequest.Prompt.SessionID ||
|
||||||
!reflect.DeepEqual(req.StructuredOutput, initialRequest.StructuredOutput) {
|
!reflect.DeepEqual(req.StructuredOutput, initialRequest.StructuredOutput) {
|
||||||
@@ -2314,8 +2326,19 @@ func TestRunnerRepairStateMachine(t *testing.T) {
|
|||||||
!reflect.DeepEqual(generated.StructuredOutput, initialRequest.StructuredOutput) {
|
!reflect.DeepEqual(generated.StructuredOutput, initialRequest.StructuredOutput) {
|
||||||
t.Fatalf("repair generation request %d common fields drifted: %+v", index, generated)
|
t.Fatalf("repair generation request %d common fields drifted: %+v", index, generated)
|
||||||
}
|
}
|
||||||
if reflect.DeepEqual(generated.Prompt.Messages, initialRequest.Prompt.Messages) {
|
expectedMessages := len(initialRequest.Prompt.Messages) + 1
|
||||||
t.Fatalf("repair generation request %d reused the initial prompt", index)
|
if strings.TrimSpace(tc.responses[index].Content) != "" {
|
||||||
|
expectedMessages++
|
||||||
|
}
|
||||||
|
if len(generated.Prompt.Messages) != expectedMessages ||
|
||||||
|
generated.Prompt.Messages[len(generated.Prompt.Messages)-1].Role != "user" {
|
||||||
|
t.Fatalf("repair generation request %d messages = %#v", index, generated.Prompt.Messages)
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(tc.responses[index].Content) != "" {
|
||||||
|
assistant := generated.Prompt.Messages[len(generated.Prompt.Messages)-2]
|
||||||
|
if assistant.Role != "assistant" || assistant.Content != tc.responses[index].Content {
|
||||||
|
t.Fatalf("repair generation request %d candidate = %+v", index, assistant)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -174,9 +174,8 @@ func validateArtifact(ctx context.Context, artifact *domain.Artifact, contract d
|
|||||||
}
|
}
|
||||||
|
|
||||||
res := domain.ValidationResult{
|
res := domain.ValidationResult{
|
||||||
Mode: contract.ValidationMode,
|
Mode: contract.ValidationMode,
|
||||||
SchemaPath: contract.SchemaPath,
|
SchemaPath: contract.SchemaPath,
|
||||||
RepairAttempts: contract.RepairAttempts,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if artifact == nil {
|
if artifact == nil {
|
||||||
|
|||||||
@@ -64,6 +64,43 @@ func TestStandardValidatorBasicFailureEmpty(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestStandardValidatorReportsZeroRepairAttempts(t *testing.T) {
|
||||||
|
v := NewStandardValidator("")
|
||||||
|
|
||||||
|
for _, tc := range []struct {
|
||||||
|
name string
|
||||||
|
body string
|
||||||
|
contract domain.OutputContract
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "passed basic validation",
|
||||||
|
body: "answer",
|
||||||
|
contract: domain.OutputContract{
|
||||||
|
ValidationMode: domain.ValidationBasic,
|
||||||
|
RepairAttempts: 3,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "failed JSON validation",
|
||||||
|
body: `{"answer":`,
|
||||||
|
contract: domain.OutputContract{
|
||||||
|
ValidationMode: domain.ValidationJSON,
|
||||||
|
RepairAttempts: 3,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
res, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(tc.body)}, tc.contract)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("validate: %v", err)
|
||||||
|
}
|
||||||
|
if res.RepairAttempts != 0 {
|
||||||
|
t.Fatalf("repair attempts = %d, want 0", res.RepairAttempts)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestStandardValidatorJSONSuccess(t *testing.T) {
|
func TestStandardValidatorJSONSuccess(t *testing.T) {
|
||||||
v := NewStandardValidator("")
|
v := NewStandardValidator("")
|
||||||
|
|
||||||
|
|||||||
@@ -22,27 +22,52 @@ func TestPreparationRejectsInvalidOutputContractWithPublicError(t *testing.T) {
|
|||||||
t.Fatalf("construct engine: %v", err)
|
t.Fatalf("construct engine: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
req := promptkit.RunRequest{
|
for _, tc := range []struct {
|
||||||
PromptID: "prompt",
|
name string
|
||||||
Validation: &promptkit.OutputContract{
|
contract promptkit.OutputContract
|
||||||
Format: promptkit.OutputFormat("binary"),
|
}{
|
||||||
ValidationMode: promptkit.ValidationNone,
|
{
|
||||||
|
name: "unsupported format",
|
||||||
|
contract: promptkit.OutputContract{
|
||||||
|
Format: promptkit.OutputFormat("binary"),
|
||||||
|
ValidationMode: promptkit.ValidationNone,
|
||||||
|
},
|
||||||
},
|
},
|
||||||
}
|
{
|
||||||
|
name: "repair attempts above maximum",
|
||||||
|
contract: promptkit.OutputContract{
|
||||||
|
Format: promptkit.FormatText,
|
||||||
|
ValidationMode: promptkit.ValidationBasic,
|
||||||
|
RepairAttempts: 4,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "none validation with repair attempts",
|
||||||
|
contract: promptkit.OutputContract{
|
||||||
|
Format: promptkit.FormatText,
|
||||||
|
ValidationMode: promptkit.ValidationNone,
|
||||||
|
RepairAttempts: 1,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
} {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
req := promptkit.RunRequest{PromptID: "prompt", Validation: &tc.contract}
|
||||||
|
|
||||||
prepared, err := engine.Prepare(context.Background(), req)
|
prepared, err := engine.Prepare(context.Background(), req)
|
||||||
if prepared != nil {
|
if prepared != nil {
|
||||||
t.Fatalf("expected no partial prepared run, got %+v", prepared)
|
t.Fatalf("expected no partial prepared run, got %+v", prepared)
|
||||||
}
|
}
|
||||||
if !errors.Is(err, promptkit.ErrInvalidRequest) {
|
if !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
t.Fatalf("prepare error = %v, want ErrInvalidRequest", err)
|
t.Fatalf("prepare error = %v, want ErrInvalidRequest", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
preparedExecution, err := engine.PrepareExecution(context.Background(), req)
|
preparedExecution, err := engine.PrepareExecution(context.Background(), req)
|
||||||
if preparedExecution != nil {
|
if preparedExecution != nil {
|
||||||
t.Fatalf("expected no partial prepared execution, got %+v", preparedExecution)
|
t.Fatalf("expected no partial prepared execution, got %+v", preparedExecution)
|
||||||
}
|
}
|
||||||
if !errors.Is(err, promptkit.ErrInvalidRequest) {
|
if !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
t.Fatalf("prepare execution error = %v, want ErrInvalidRequest", err)
|
t.Fatalf("prepare execution error = %v, want ErrInvalidRequest", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
|
||||||
"reflect"
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -257,6 +256,50 @@ func TestPreparedExecutionLifecycleAndEngineBinding(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPreparedExecutionRepairsEmptyBasicOutput(t *testing.T) {
|
||||||
|
client := &preparedRecordingClient{responses: []*promptkit.GenerateResponse{
|
||||||
|
{
|
||||||
|
Content: "",
|
||||||
|
Usage: promptkit.TokenUsage{PromptTokens: 3, CompletionTokens: 5, TotalTokens: 8},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Content: "Corrected summary.",
|
||||||
|
Usage: promptkit.TokenUsage{PromptTokens: 7, CompletionTokens: 11, TotalTokens: 18},
|
||||||
|
},
|
||||||
|
}}
|
||||||
|
engine := newPreparedContractEngine(t, client, "Summarize the source.")
|
||||||
|
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
||||||
|
PromptID: "prepared",
|
||||||
|
Validation: &promptkit.OutputContract{
|
||||||
|
Format: promptkit.FormatMarkdown,
|
||||||
|
ValidationMode: promptkit.ValidationBasic,
|
||||||
|
RepairAttempts: 1,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution: %v", err)
|
||||||
|
}
|
||||||
|
details := prepared.Details()
|
||||||
|
|
||||||
|
result, err := engine.RunPrepared(context.Background(), prepared)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("run prepared: %v", err)
|
||||||
|
}
|
||||||
|
if result.RawOutput != "Corrected summary." || result.Validation.Status != promptkit.ValidationPassed ||
|
||||||
|
result.Validation.RepairAttempts != 1 || result.Usage != (promptkit.TokenUsage{PromptTokens: 10, CompletionTokens: 16, TotalTokens: 26}) {
|
||||||
|
t.Fatalf("repaired result = %+v", result)
|
||||||
|
}
|
||||||
|
requests := client.snapshot()
|
||||||
|
if len(requests) != 2 || len(requests[1].Prompt.Messages) != len(details.Messages)+1 ||
|
||||||
|
!reflect.DeepEqual(requests[1].Prompt.Messages[:len(details.Messages)], details.Messages) ||
|
||||||
|
requests[1].Prompt.Messages[len(requests[1].Prompt.Messages)-1].Role != "user" {
|
||||||
|
t.Fatalf("prepared repair requests = %#v", requests)
|
||||||
|
}
|
||||||
|
if _, err := engine.RunPrepared(context.Background(), prepared); !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("second RunPrepared error = %v, want ErrInvalidRequest", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestPreparedExecutionConcurrentClaimAllowsOneGeneration(t *testing.T) {
|
func TestPreparedExecutionConcurrentClaimAllowsOneGeneration(t *testing.T) {
|
||||||
release := make(chan struct{})
|
release := make(chan struct{})
|
||||||
client := &preparedRecordingClient{
|
client := &preparedRecordingClient{
|
||||||
@@ -524,19 +567,25 @@ func TestPreparedExecutionCredentialCapacityAndTimingBoundaries(t *testing.T) {
|
|||||||
engine, err := promptkit.NewEngine(
|
engine, err := promptkit.NewEngine(
|
||||||
promptkit.Config{},
|
promptkit.Config{},
|
||||||
promptkit.WithPromptFS(contractPromptFS("prepared", "profile", "content"), "."),
|
promptkit.WithPromptFS(contractPromptFS("prepared", "profile", "content"), "."),
|
||||||
promptkit.WithProfileFS(preparedCredentialProfileSource(environmentName), "."),
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "profile",
|
||||||
|
Endpoint: "http://example.test/v1",
|
||||||
|
Model: "model",
|
||||||
|
APIKeyRequired: true,
|
||||||
|
}),
|
||||||
promptkit.WithLLMClient(client),
|
promptkit.WithLLMClient(client),
|
||||||
)
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("construct credential engine: %v", err)
|
t.Fatalf("construct credential engine: %v", err)
|
||||||
}
|
}
|
||||||
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
|
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
||||||
|
PromptID: "prepared",
|
||||||
|
Execution: &promptkit.ExecutionTargetOverride{APIKeyEnv: environmentName},
|
||||||
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("prepare credential execution: %v", err)
|
t.Fatalf("prepare credential execution: %v", err)
|
||||||
}
|
}
|
||||||
if err := os.Unsetenv(environmentName); err != nil {
|
t.Setenv(environmentName, "")
|
||||||
t.Fatalf("unset credential environment: %v", err)
|
|
||||||
}
|
|
||||||
|
|
||||||
result, err := engine.RunPrepared(context.Background(), prepared)
|
result, err := engine.RunPrepared(context.Background(), prepared)
|
||||||
if result != nil ||
|
if result != nil ||
|
||||||
@@ -666,12 +715,13 @@ func (r *mutablePreparedArtifactReader) callCount() int {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type preparedRecordingClient struct {
|
type preparedRecordingClient struct {
|
||||||
mu sync.Mutex
|
mu sync.Mutex
|
||||||
response *promptkit.GenerateResponse
|
response *promptkit.GenerateResponse
|
||||||
err error
|
responses []*promptkit.GenerateResponse
|
||||||
requests []promptkit.GenerateRequest
|
err error
|
||||||
started chan struct{}
|
requests []promptkit.GenerateRequest
|
||||||
release <-chan struct{}
|
started chan struct{}
|
||||||
|
release <-chan struct{}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *preparedRecordingClient) Generate(
|
func (c *preparedRecordingClient) Generate(
|
||||||
@@ -695,6 +745,12 @@ func (c *preparedRecordingClient) Generate(
|
|||||||
if c.err != nil {
|
if c.err != nil {
|
||||||
return nil, c.err
|
return nil, c.err
|
||||||
}
|
}
|
||||||
|
if len(c.responses) > 0 {
|
||||||
|
if index := len(c.requests) - 1; index < len(c.responses) {
|
||||||
|
return c.responses[index], nil
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("no response configured for generation %d", len(c.requests))
|
||||||
|
}
|
||||||
return c.response, nil
|
return c.response, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -755,16 +811,6 @@ model: ` + model + `
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func preparedCredentialProfileSource(environmentName string) fstest.MapFS {
|
|
||||||
return fstest.MapFS{
|
|
||||||
"profile.yaml": &fstest.MapFile{Data: []byte(`id: profile
|
|
||||||
endpoint: http://example.test/v1
|
|
||||||
model: model
|
|
||||||
api_key_env: ` + environmentName + `
|
|
||||||
`)},
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func preparedSchemaSource() fstest.MapFS {
|
func preparedSchemaSource() fstest.MapFS {
|
||||||
return fstest.MapFS{
|
return fstest.MapFS{
|
||||||
"schema.json": &fstest.MapFile{Data: []byte(`{
|
"schema.json": &fstest.MapFile{Data: []byte(`{
|
||||||
|
|||||||
243
profile_inheritance_contract_test.go
Normal file
243
profile_inheritance_contract_test.go
Normal file
@@ -0,0 +1,243 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"io/fs"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestProfileInheritanceBuiltInAliasWorkflow(t *testing.T) {
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
|
PromptDir: frameworkPromptDir,
|
||||||
|
SchemaDir: frameworkSchemaDir,
|
||||||
|
}, promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "weather-light",
|
||||||
|
BaseProfileID: "deepseek-4-flash",
|
||||||
|
ReasoningEffort: "high",
|
||||||
|
TimeoutSeconds: 120,
|
||||||
|
}))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct alias engine: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
base, err := engine.InspectProfile(context.Background(), "deepseek-4-flash")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("inspect base: %v", err)
|
||||||
|
}
|
||||||
|
child, err := engine.InspectProfile(context.Background(), "weather-light")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("inspect alias: %v", err)
|
||||||
|
}
|
||||||
|
if child.ProfileID != "weather-light" ||
|
||||||
|
child.EffectiveModelParams.BackendID != base.EffectiveModelParams.BackendID ||
|
||||||
|
child.EffectiveModelParams.Model != base.EffectiveModelParams.Model ||
|
||||||
|
child.EffectiveModelParams.ReasoningEffort != "high" ||
|
||||||
|
child.EffectiveModelParams.TimeoutSeconds != 120 {
|
||||||
|
t.Fatalf("alias inspection = %+v, base = %+v", child, base)
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||||
|
PromptID: frameworkMarkdownSummaryPromptID,
|
||||||
|
ProfileID: "weather-light",
|
||||||
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
|
"transcript": promptkit.Inline("Rin opens the gate."),
|
||||||
|
"glossary": promptkit.Inline("gate: A guarded passage."),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare alias: %v", err)
|
||||||
|
}
|
||||||
|
if prepared.SelectedProfileID != "weather-light" {
|
||||||
|
t.Fatalf("SelectedProfileID = %q", prepared.SelectedProfileID)
|
||||||
|
}
|
||||||
|
|
||||||
|
timeout := 15
|
||||||
|
reasoning := "low"
|
||||||
|
overridden, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||||
|
PromptID: frameworkMarkdownSummaryPromptID,
|
||||||
|
ProfileID: "weather-light",
|
||||||
|
Execution: &promptkit.ExecutionTargetOverride{
|
||||||
|
TimeoutSeconds: &timeout,
|
||||||
|
ReasoningEffort: &reasoning,
|
||||||
|
},
|
||||||
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
|
"transcript": promptkit.Inline("Rin opens the gate."),
|
||||||
|
"glossary": promptkit.Inline("gate: A guarded passage."),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare override: %v", err)
|
||||||
|
}
|
||||||
|
if overridden.EffectiveModelParams.TimeoutSeconds != timeout ||
|
||||||
|
overridden.EffectiveModelParams.ReasoningEffort != reasoning {
|
||||||
|
t.Fatalf("runtime override target = %+v", overridden.EffectiveModelParams)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProfileInheritanceYAMLAliasOfBuiltIn(t *testing.T) {
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(fstest.MapFS{}, "."),
|
||||||
|
promptkit.WithProfileFS(fstest.MapFS{
|
||||||
|
"alias.yaml": &fstest.MapFile{Data: []byte("id: yaml-alias\nbase_profile: deepseek-4-flash\n")},
|
||||||
|
}, "."),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct YAML alias engine: %v", err)
|
||||||
|
}
|
||||||
|
inspection, err := engine.InspectProfile(context.Background(), "yaml-alias")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("inspect YAML alias: %v", err)
|
||||||
|
}
|
||||||
|
if inspection.ProfileID != "yaml-alias" || inspection.EffectiveModelParams.Model == "" {
|
||||||
|
t.Fatalf("YAML alias inspection = %+v", inspection)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProfileInheritanceRetainsRequiredCredentialBehavior(t *testing.T) {
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(fstest.MapFS{}, "."),
|
||||||
|
promptkit.WithBackend(promptkit.Backend{
|
||||||
|
ID: "credential-backend",
|
||||||
|
Endpoint: "https://credential.example/v1",
|
||||||
|
APIKeyEnv: "OPTIONAL_BACKEND_KEY",
|
||||||
|
}),
|
||||||
|
promptkit.WithProfiles(
|
||||||
|
promptkit.Profile{
|
||||||
|
ID: "credential-base",
|
||||||
|
BackendID: "credential-backend",
|
||||||
|
Model: "model",
|
||||||
|
APIKeyRequired: true,
|
||||||
|
},
|
||||||
|
promptkit.Profile{ID: "credential-child", BaseProfileID: "credential-base"},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct credential inheritance engine: %v", err)
|
||||||
|
}
|
||||||
|
inspection, err := engine.InspectProfile(context.Background(), "credential-child")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("inspect credential child: %v", err)
|
||||||
|
}
|
||||||
|
if !inspection.APIKeyRequired || inspection.EffectiveModelParams.APIKeyEnv != "" {
|
||||||
|
t.Fatalf("credential inspection = %+v", inspection)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProfileInheritancePreservesPublicErrorIdentities(t *testing.T) {
|
||||||
|
newEngine := func(t *testing.T, profiles fs.FS) *promptkit.Engine {
|
||||||
|
t.Helper()
|
||||||
|
options := []promptkit.Option{promptkit.WithPromptFS(fstest.MapFS{}, ".")}
|
||||||
|
if profiles != nil {
|
||||||
|
options = append(options, promptkit.WithProfileFS(profiles, "."))
|
||||||
|
}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{}, options...)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine: %v", err)
|
||||||
|
}
|
||||||
|
return engine
|
||||||
|
}
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
profiles fs.FS
|
||||||
|
profile string
|
||||||
|
contains []string
|
||||||
|
want error
|
||||||
|
wantNot error
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "missing selected profile",
|
||||||
|
profile: "missing",
|
||||||
|
want: promptkit.ErrProfileNotFound,
|
||||||
|
wantNot: promptkit.ErrProfileLoad,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "missing base",
|
||||||
|
profiles: fstest.MapFS{
|
||||||
|
"child.yaml": &fstest.MapFile{Data: []byte("id: child\nbase_profile: missing\n")},
|
||||||
|
},
|
||||||
|
profile: "child",
|
||||||
|
contains: []string{"child", "missing"},
|
||||||
|
want: promptkit.ErrProfileLoad,
|
||||||
|
wantNot: promptkit.ErrProfileNotFound,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "cycle",
|
||||||
|
profiles: fstest.MapFS{
|
||||||
|
"a.yaml": &fstest.MapFile{Data: []byte("id: a\nbase_profile: b\n")},
|
||||||
|
"b.yaml": &fstest.MapFile{Data: []byte("id: b\nbase_profile: a\n")},
|
||||||
|
},
|
||||||
|
profile: "a",
|
||||||
|
contains: []string{"a", "b"},
|
||||||
|
want: promptkit.ErrProfileLoad,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
result, err := newEngine(t, tc.profiles).InspectProfile(context.Background(), tc.profile)
|
||||||
|
if result != nil || !errors.Is(err, tc.want) || (tc.wantNot != nil && errors.Is(err, tc.wantNot)) {
|
||||||
|
t.Fatalf("inspection=(%+v, %v), want %v without %v", result, err, tc.want, tc.wantNot)
|
||||||
|
}
|
||||||
|
for _, fragment := range tc.contains {
|
||||||
|
if !strings.Contains(err.Error(), fragment) {
|
||||||
|
t.Fatalf("error = %v, want %q", err, fragment)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestProfileInheritanceFreezesPreparedExecution(t *testing.T) {
|
||||||
|
profiles := &mutableInheritanceProfileFS{files: fstest.MapFS{
|
||||||
|
"child.yaml": &fstest.MapFile{Data: []byte("id: child\nbase_profile: base\n")},
|
||||||
|
"base.yaml": &fstest.MapFile{Data: []byte("id: base\nendpoint: https://base.example/v1\nmodel: first-model\n")},
|
||||||
|
}}
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prepared", "child", "content"), "."),
|
||||||
|
promptkit.WithProfileFS(profiles, "."),
|
||||||
|
promptkit.WithLLMClient(client),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution: %v", err)
|
||||||
|
}
|
||||||
|
profiles.set("base.yaml", "id: base\nendpoint: https://base.example/v1\nmodel: second-model\n")
|
||||||
|
|
||||||
|
result, err := engine.RunPrepared(context.Background(), prepared)
|
||||||
|
if err != nil || result == nil || len(client.requests) != 1 || client.requests[0].Target.Model != "first-model" {
|
||||||
|
t.Fatalf("prepared execution=(%+v, %v), requests=%+v", result, err, client.requests)
|
||||||
|
}
|
||||||
|
inspection, err := engine.InspectProfile(context.Background(), "child")
|
||||||
|
if err != nil || inspection.EffectiveModelParams.Model != "second-model" {
|
||||||
|
t.Fatalf("fresh inspection=(%+v, %v)", inspection, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
type mutableInheritanceProfileFS struct {
|
||||||
|
mu sync.RWMutex
|
||||||
|
files fstest.MapFS
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *mutableInheritanceProfileFS) Open(name string) (fs.File, error) {
|
||||||
|
f.mu.RLock()
|
||||||
|
defer f.mu.RUnlock()
|
||||||
|
return f.files.Open(name)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f *mutableInheritanceProfileFS) set(name, content string) {
|
||||||
|
f.mu.Lock()
|
||||||
|
defer f.mu.Unlock()
|
||||||
|
f.files[name] = &fstest.MapFile{Data: []byte(content)}
|
||||||
|
}
|
||||||
41
profiles.go
41
profiles.go
@@ -2,17 +2,16 @@ package promptkit
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||||
)
|
)
|
||||||
|
|
||||||
// OpenAICompatibleProfile returns an ordinary in-memory Profile for an
|
// OpenAICompatibleProfile returns an in-memory Profile for an OpenAI-compatible
|
||||||
// OpenAI-compatible chat-completions endpoint.
|
// chat-completions endpoint. A non-blank BaseProfileID permits its target
|
||||||
|
// fields to be inherited when the profile is selected or inspected.
|
||||||
//
|
//
|
||||||
// It does not register global state, maintain a model catalog, or resolve
|
// It does not register global state, maintain a model catalog, or resolve
|
||||||
// credentials. If APIKeyRequired is true, callers satisfy it with
|
// credentials. If APIKeyRequired is true, callers satisfy it with
|
||||||
@@ -25,6 +24,7 @@ import (
|
|||||||
func OpenAICompatibleProfile(cfg OpenAICompatibleProfileConfig) Profile {
|
func OpenAICompatibleProfile(cfg OpenAICompatibleProfileConfig) Profile {
|
||||||
return Profile{
|
return Profile{
|
||||||
ID: cfg.ID,
|
ID: cfg.ID,
|
||||||
|
BaseProfileID: cfg.BaseProfileID,
|
||||||
BackendID: cfg.BackendID,
|
BackendID: cfg.BackendID,
|
||||||
Endpoint: cfg.Endpoint,
|
Endpoint: cfg.Endpoint,
|
||||||
Model: cfg.Model,
|
Model: cfg.Model,
|
||||||
@@ -87,8 +87,9 @@ func toDomainProfile(publicProfile Profile) (domain.ExecutionProfile, error) {
|
|||||||
return domain.ExecutionProfile{}, err
|
return domain.ExecutionProfile{}, err
|
||||||
}
|
}
|
||||||
prof := domain.ExecutionProfile{
|
prof := domain.ExecutionProfile{
|
||||||
ID: strings.TrimSpace(publicProfile.ID),
|
ID: publicProfile.ID,
|
||||||
BackendID: strings.TrimSpace(publicProfile.BackendID),
|
BaseProfileID: publicProfile.BaseProfileID,
|
||||||
|
BackendID: publicProfile.BackendID,
|
||||||
Endpoint: publicProfile.Endpoint,
|
Endpoint: publicProfile.Endpoint,
|
||||||
Model: publicProfile.Model,
|
Model: publicProfile.Model,
|
||||||
Temperature: publicProfile.Temperature,
|
Temperature: publicProfile.Temperature,
|
||||||
@@ -100,34 +101,8 @@ func toDomainProfile(publicProfile Profile) (domain.ExecutionProfile, error) {
|
|||||||
APIKeyRequired: publicProfile.APIKeyRequired,
|
APIKeyRequired: publicProfile.APIKeyRequired,
|
||||||
ExtraParams: extraParams,
|
ExtraParams: extraParams,
|
||||||
}
|
}
|
||||||
if err := normalizeAndValidatePublicProfile(&prof); err != nil {
|
if err := profile.NormalizeAndValidateDefinition(&prof); err != nil {
|
||||||
return domain.ExecutionProfile{}, err
|
return domain.ExecutionProfile{}, err
|
||||||
}
|
}
|
||||||
return prof, nil
|
return prof, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func normalizeAndValidatePublicProfile(prof *domain.ExecutionProfile) error {
|
|
||||||
if strings.TrimSpace(prof.ID) == "" {
|
|
||||||
return errors.New("id is required")
|
|
||||||
}
|
|
||||||
prof.Endpoint = strings.TrimSpace(prof.Endpoint)
|
|
||||||
if strings.TrimSpace(prof.BackendID) == "" && prof.Endpoint == "" {
|
|
||||||
return errors.New("backend or endpoint is required")
|
|
||||||
}
|
|
||||||
if prof.Endpoint != "" {
|
|
||||||
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(prof.Endpoint)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
prof.Endpoint = endpoint
|
|
||||||
}
|
|
||||||
if strings.TrimSpace(prof.Model) == "" {
|
|
||||||
return errors.New("model is required")
|
|
||||||
}
|
|
||||||
return domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{
|
|
||||||
Temperature: prof.Temperature,
|
|
||||||
MaxTokens: prof.MaxTokens,
|
|
||||||
TopP: prof.TopP,
|
|
||||||
TimeoutSeconds: prof.TimeoutSeconds,
|
|
||||||
})
|
|
||||||
}
|
|
||||||
|
|||||||
@@ -844,7 +844,7 @@ func TestBackendExtraParamsAreDeeplyCopiedAtConstructionAndLookup(t *testing.T)
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestEngineValidationIsSinglePass(t *testing.T) {
|
func TestEngineValidationWithZeroRepairBudgetIsSinglePass(t *testing.T) {
|
||||||
client := &fakeLLMClient{
|
client := &fakeLLMClient{
|
||||||
response: &promptkit.GenerateResponse{Content: "not-json"},
|
response: &promptkit.GenerateResponse{Content: "not-json"},
|
||||||
}
|
}
|
||||||
@@ -859,7 +859,7 @@ func TestEngineValidationIsSinglePass(t *testing.T) {
|
|||||||
Validation: &promptkit.OutputContract{
|
Validation: &promptkit.OutputContract{
|
||||||
Format: promptkit.FormatJSON,
|
Format: promptkit.FormatJSON,
|
||||||
ValidationMode: promptkit.ValidationJSON,
|
ValidationMode: promptkit.ValidationJSON,
|
||||||
RepairAttempts: 3,
|
RepairAttempts: 0,
|
||||||
},
|
},
|
||||||
})
|
})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -874,6 +874,87 @@ func TestEngineValidationIsSinglePass(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestEngineRunRepairsJSONSchemaOutput(t *testing.T) {
|
||||||
|
client := &fakeLLMClient{responses: []*promptkit.GenerateResponse{
|
||||||
|
{
|
||||||
|
Content: "{}",
|
||||||
|
Usage: promptkit.TokenUsage{PromptTokens: 2, CompletionTokens: 3, TotalTokens: 5, CachedTokens: 7, CacheWriteTokens: 11},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Content: `{"events":[{"title":"Repaired event"}]}`,
|
||||||
|
Usage: promptkit.TokenUsage{PromptTokens: 13, CompletionTokens: 17, TotalTokens: 19, CachedTokens: 23, CacheWriteTokens: 29},
|
||||||
|
},
|
||||||
|
}}
|
||||||
|
engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(client))
|
||||||
|
|
||||||
|
result, err := engine.Run(context.Background(), promptkit.RunRequest{
|
||||||
|
PromptID: frameworkStructuredEventsPromptID,
|
||||||
|
SessionID: " repair-session ",
|
||||||
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
|
"transcript": promptkit.Inline("Rin opens the gate."),
|
||||||
|
"glossary": promptkit.Inline("gate: A guarded passage."),
|
||||||
|
},
|
||||||
|
Validation: &promptkit.OutputContract{
|
||||||
|
Format: promptkit.FormatJSON,
|
||||||
|
ValidationMode: promptkit.ValidationJSONSchema,
|
||||||
|
SchemaPath: "structured_events.schema.json",
|
||||||
|
RepairAttempts: 1,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("run: %v", err)
|
||||||
|
}
|
||||||
|
if result.RawOutput != client.responses[1].Content || result.Validation.Status != promptkit.ValidationPassed ||
|
||||||
|
result.Validation.RepairAttempts != 1 {
|
||||||
|
t.Fatalf("repaired result = %+v", result)
|
||||||
|
}
|
||||||
|
wantUsage := promptkit.TokenUsage{PromptTokens: 15, CompletionTokens: 20, TotalTokens: 24, CachedTokens: 30, CacheWriteTokens: 40}
|
||||||
|
if result.Usage != wantUsage {
|
||||||
|
t.Fatalf("usage = %+v, want %+v", result.Usage, wantUsage)
|
||||||
|
}
|
||||||
|
if len(client.requests) != 2 {
|
||||||
|
t.Fatalf("generation calls = %d, want 2", len(client.requests))
|
||||||
|
}
|
||||||
|
initial, repaired := client.requests[0], client.requests[1]
|
||||||
|
if initial.Prompt.SessionID != "repair-session" || repaired.Prompt.SessionID != initial.Prompt.SessionID ||
|
||||||
|
!reflect.DeepEqual(repaired.Target, initial.Target) || repaired.TargetPresence != initial.TargetPresence ||
|
||||||
|
!reflect.DeepEqual(repaired.StructuredOutput, initial.StructuredOutput) {
|
||||||
|
t.Fatalf("generation request state drifted: initial=%+v repaired=%+v", initial, repaired)
|
||||||
|
}
|
||||||
|
if initial.StructuredOutput == nil || initial.StructuredOutput.JSONSchema == nil {
|
||||||
|
t.Fatalf("expected structured output on initial request: %+v", initial)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEngineRunReturnsFinalResultAfterRepairExhaustion(t *testing.T) {
|
||||||
|
client := &fakeLLMClient{responses: []*promptkit.GenerateResponse{
|
||||||
|
{Content: "not-json", Usage: promptkit.TokenUsage{PromptTokens: 2, CompletionTokens: 3, TotalTokens: 5}},
|
||||||
|
{Content: "still-not-json", Usage: promptkit.TokenUsage{PromptTokens: 7, CompletionTokens: 11, TotalTokens: 18}},
|
||||||
|
}}
|
||||||
|
engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(client))
|
||||||
|
|
||||||
|
result, err := engine.Run(context.Background(), promptkit.RunRequest{
|
||||||
|
PromptID: frameworkMarkdownSummaryPromptID,
|
||||||
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
|
"transcript": promptkit.Inline("Rin opens the gate."),
|
||||||
|
"glossary": promptkit.Inline("gate: A guarded passage."),
|
||||||
|
},
|
||||||
|
Validation: &promptkit.OutputContract{
|
||||||
|
Format: promptkit.FormatJSON,
|
||||||
|
ValidationMode: promptkit.ValidationJSON,
|
||||||
|
RepairAttempts: 1,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil || result == nil {
|
||||||
|
t.Fatalf("run = (%+v, %v), want exhausted result", result, err)
|
||||||
|
}
|
||||||
|
if result.RawOutput != "still-not-json" || result.Validation.Status != promptkit.ValidationFailed ||
|
||||||
|
result.Validation.RepairAttempts != 1 || len(result.Validation.Errors) == 0 ||
|
||||||
|
result.Usage != (promptkit.TokenUsage{PromptTokens: 9, CompletionTokens: 14, TotalTokens: 23}) {
|
||||||
|
t.Fatalf("exhausted result = %+v", result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) {
|
func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) {
|
||||||
profile := promptkit.Profile{ID: "profile", Endpoint: "http://example.test/v1", Model: "model"}
|
profile := promptkit.Profile{ID: "profile", Endpoint: "http://example.test/v1", Model: "model"}
|
||||||
|
|
||||||
|
|||||||
81
types.go
81
types.go
@@ -322,7 +322,10 @@ type ExecutionTarget struct {
|
|||||||
// ReasoningEffort is the effective opaque provider-specific reasoning
|
// ReasoningEffort is the effective opaque provider-specific reasoning
|
||||||
// setting. An empty value instructs model clients to omit reasoning.
|
// setting. An empty value instructs model clients to omit reasoning.
|
||||||
ReasoningEffort string `json:"reasoning_effort"`
|
ReasoningEffort string `json:"reasoning_effort"`
|
||||||
// APIKeyEnv is an environment-variable name, not its credential value.
|
// APIKeyEnv is the resolved name of an optional environment lookup source,
|
||||||
|
// not its credential value. The built-in client omits Authorization when no
|
||||||
|
// usable direct or environment credential is available; injected clients may
|
||||||
|
// resolve this metadata differently.
|
||||||
APIKeyEnv string `json:"api_key_env"`
|
APIKeyEnv string `json:"api_key_env"`
|
||||||
// ExtraParams contains copied JSON-compatible provider parameters.
|
// ExtraParams contains copied JSON-compatible provider parameters.
|
||||||
ExtraParams map[string]any `json:"extra_params"`
|
ExtraParams map[string]any `json:"extra_params"`
|
||||||
@@ -426,9 +429,10 @@ type ExecutionTargetOverride struct {
|
|||||||
// inherited value and disables reasoning for this run. Non-blank values
|
// inherited value and disables reasoning for this run. Non-blank values
|
||||||
// are opaque and are not validated against a fixed vocabulary.
|
// are opaque and are not validated against a fixed vocabulary.
|
||||||
ReasoningEffort *string
|
ReasoningEffort *string
|
||||||
// APIKeyEnv replaces the profile or backend environment-variable name when
|
// APIKeyEnv replaces the profile or backend optional environment lookup
|
||||||
// non-blank. A direct RunRequest.APIKey still takes precedence over
|
// source when non-blank. A direct RunRequest.APIKey still takes precedence.
|
||||||
// environment lookup.
|
// The built-in client omits Authorization when neither source has a usable
|
||||||
|
// value; injected clients may resolve this metadata differently.
|
||||||
APIKeyEnv string
|
APIKeyEnv string
|
||||||
// ExtraParams, when non-empty, replaces the complete profile or backend map.
|
// ExtraParams, when non-empty, replaces the complete profile or backend map.
|
||||||
// Values must be JSON-compatible: nil, booleans, finite numbers, strings,
|
// Values must be JSON-compatible: nil, booleans, finite numbers, strings,
|
||||||
@@ -439,31 +443,43 @@ type ExecutionTargetOverride struct {
|
|||||||
|
|
||||||
// Profile is an in-memory execution profile for library consumers.
|
// Profile is an in-memory execution profile for library consumers.
|
||||||
//
|
//
|
||||||
// It is equivalent to a loaded profile file after validation. Raw API keys do
|
// A standalone Profile is equivalent to a loaded profile file after local
|
||||||
// not belong in profiles; use APIKeyRequired to require callers to provide a
|
// validation. A derived profile names BaseProfileID and can inherit target
|
||||||
// RunRequest.APIKey or explicit request ExecutionTargetOverride.APIKeyEnv, or
|
// fields when selected or inspected. Raw API keys do not belong in profiles;
|
||||||
// use profile YAML api_key_env with file and FS profile sources. Profile has no
|
// use APIKeyRequired to require callers to provide a RunRequest.APIKey or
|
||||||
// stable JSON representation.
|
// explicit request ExecutionTargetOverride.APIKeyEnv, or use profile YAML
|
||||||
|
// api_key_env with file and FS profile sources. Profile has no stable JSON
|
||||||
|
// representation.
|
||||||
//
|
//
|
||||||
// WithProfiles validates and copies Profile values during NewEngine. Zero
|
// WithProfiles locally validates and copies Profile values during NewEngine.
|
||||||
// Temperature, MaxTokens, and TopP values and blank ServiceTier and
|
// It checks base-reference existence and resolved target completeness when a
|
||||||
// ReasoningEffort values leave those provider controls unspecified. A zero
|
// derived profile is selected or inspected. Zero Temperature, MaxTokens, and
|
||||||
// TimeoutSeconds retains the framework deadline, while an empty ExtraParams map
|
// TopP values and blank ServiceTier and ReasoningEffort values leave those
|
||||||
// inherits backend request defaults. Use ExecutionTargetOverride pointer fields
|
// provider controls unspecified. A zero TimeoutSeconds retains the framework
|
||||||
// to request an explicit numeric zero.
|
// deadline, while an empty ExtraParams map inherits backend request defaults.
|
||||||
|
// Use ExecutionTargetOverride pointer fields to request an explicit numeric
|
||||||
|
// zero.
|
||||||
type Profile struct {
|
type Profile struct {
|
||||||
// ID is the required non-blank profile identifier. WithProfiles trims it.
|
// ID is the required non-blank profile identifier. WithProfiles trims it.
|
||||||
ID string
|
ID string
|
||||||
|
// BaseProfileID optionally names one base profile. WithProfiles trims it. A
|
||||||
|
// non-blank value permits required target fields to be inherited when the
|
||||||
|
// profile is selected or inspected, which is also when reference existence
|
||||||
|
// and resolved completeness are checked. A blank value leaves this as a
|
||||||
|
// standalone profile.
|
||||||
|
BaseProfileID string
|
||||||
// BackendID optionally selects an engine backend. WithProfiles trims it.
|
// BackendID optionally selects an engine backend. WithProfiles trims it.
|
||||||
// Backend membership is checked when a request selects the profile; an
|
// Backend membership is checked when a request selects the profile; an
|
||||||
// unknown ID makes preparation fail with ErrProfileLoad.
|
// unknown ID makes preparation fail with ErrProfileLoad.
|
||||||
BackendID string
|
BackendID string
|
||||||
// Endpoint is the model-provider base URL. It is required only when
|
// Endpoint is the model-provider base URL. A standalone Profile requires an
|
||||||
// BackendID is blank and otherwise overrides the backend endpoint when
|
// endpoint when BackendID is blank; a derived Profile may inherit either
|
||||||
|
// field. A non-blank endpoint overrides the backend endpoint when
|
||||||
// non-blank. WithProfiles trims it and requires an absolute HTTP or HTTPS URL
|
// non-blank. WithProfiles trims it and requires an absolute HTTP or HTTPS URL
|
||||||
// with a host and no user information, query, or fragment.
|
// with a host and no user information, query, or fragment.
|
||||||
Endpoint string
|
Endpoint string
|
||||||
// Model is the required non-blank provider model identifier.
|
// Model is the provider model identifier. It is required for a standalone
|
||||||
|
// Profile and may be inherited by a derived Profile.
|
||||||
Model string
|
Model string
|
||||||
// Temperature is from 0 through 2. Zero leaves the provider control
|
// Temperature is from 0 through 2. Zero leaves the provider control
|
||||||
// unspecified.
|
// unspecified.
|
||||||
@@ -481,7 +497,8 @@ type Profile struct {
|
|||||||
ReasoningEffort string
|
ReasoningEffort string
|
||||||
// APIKeyRequired clears a backend's inherited API-key environment name and
|
// APIKeyRequired clears a backend's inherited API-key environment name and
|
||||||
// requires a non-blank RunRequest.APIKey unless the request explicitly
|
// requires a non-blank RunRequest.APIKey unless the request explicitly
|
||||||
// supplies ExecutionTargetOverride.APIKeyEnv. It does not store a credential.
|
// supplies ExecutionTargetOverride.APIKeyEnv. When false, a named
|
||||||
|
// environment source remains optional. It does not store a credential.
|
||||||
APIKeyRequired bool
|
APIKeyRequired bool
|
||||||
// ExtraParams contains provider-specific JSON-compatible values. An empty
|
// ExtraParams contains provider-specific JSON-compatible values. An empty
|
||||||
// map inherits backend request defaults, when any. WithProfiles validates
|
// map inherits backend request defaults, when any. WithProfiles validates
|
||||||
@@ -494,13 +511,16 @@ type Profile struct {
|
|||||||
// profile.
|
// profile.
|
||||||
//
|
//
|
||||||
// It contains ordinary profile fields for OpenAI-compatible chat-completions
|
// It contains ordinary profile fields for OpenAI-compatible chat-completions
|
||||||
// endpoints. APIKeyRequired follows Profile.APIKeyRequired. Raw API keys do not
|
// endpoints. BaseProfileID and APIKeyRequired follow Profile. Raw API keys do
|
||||||
// belong in this config. OpenAICompatibleProfileConfig has no stable JSON
|
// not belong in this config. OpenAICompatibleProfileConfig has no stable JSON
|
||||||
// representation and is not validated until its resulting Profile is supplied
|
// representation and is not validated until its resulting Profile is supplied
|
||||||
// through WithProfiles to NewEngine.
|
// through WithProfiles to NewEngine.
|
||||||
type OpenAICompatibleProfileConfig struct {
|
type OpenAICompatibleProfileConfig struct {
|
||||||
// ID becomes Profile.ID.
|
// ID becomes Profile.ID.
|
||||||
ID string
|
ID string
|
||||||
|
// BaseProfileID becomes Profile.BaseProfileID. A non-blank value permits the
|
||||||
|
// resulting Profile to inherit target fields when it is selected or inspected.
|
||||||
|
BaseProfileID string
|
||||||
// BackendID becomes Profile.BackendID.
|
// BackendID becomes Profile.BackendID.
|
||||||
BackendID string
|
BackendID string
|
||||||
// Endpoint becomes Profile.Endpoint.
|
// Endpoint becomes Profile.Endpoint.
|
||||||
@@ -545,8 +565,8 @@ type ExecutionTargetPresence struct {
|
|||||||
// JSON representation.
|
// JSON representation.
|
||||||
//
|
//
|
||||||
// A non-nil RunRequest.Validation replaces the complete prompt contract. It
|
// A non-nil RunRequest.Validation replaces the complete prompt contract. It
|
||||||
// does not merge fields. The public Engine validates generated output once and
|
// does not merge fields. The public Engine performs bounded correction after a
|
||||||
// does not install an output repairer.
|
// failed eligible validation when RepairAttempts is positive.
|
||||||
type OutputContract struct {
|
type OutputContract struct {
|
||||||
// Format selects generated artifact metadata. An empty value in a non-nil
|
// Format selects generated artifact metadata. An empty value in a non-nil
|
||||||
// request replacement defaults to FormatText.
|
// request replacement defaults to FormatText.
|
||||||
@@ -557,9 +577,9 @@ type OutputContract struct {
|
|||||||
// SchemaPath is required when ValidationMode is ValidationJSONSchema and is
|
// SchemaPath is required when ValidationMode is ValidationJSONSchema and is
|
||||||
// ignored by other modes.
|
// ignored by other modes.
|
||||||
SchemaPath string `json:"schema_path"`
|
SchemaPath string `json:"schema_path"`
|
||||||
// RepairAttempts is a non-negative requested repair limit. Zero requests no
|
// RepairAttempts is an additional generation-call budget from zero through
|
||||||
// repairs. The public Engine performs no repairs even when this value is
|
// three. Zero is single-pass. A positive value is valid only with basic,
|
||||||
// positive, so its runs report zero attempts used.
|
// json, or json_schema validation.
|
||||||
RepairAttempts int `json:"repair_attempts"`
|
RepairAttempts int `json:"repair_attempts"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -575,8 +595,8 @@ type ValidationResult struct {
|
|||||||
Errors []string `json:"errors,omitempty"`
|
Errors []string `json:"errors,omitempty"`
|
||||||
// SchemaPath is the effective schema path for JSON Schema validation.
|
// SchemaPath is the effective schema path for JSON Schema validation.
|
||||||
SchemaPath string `json:"schema_path,omitempty"`
|
SchemaPath string `json:"schema_path,omitempty"`
|
||||||
// RepairAttempts is the number of repairs actually attempted. It is always
|
// RepairAttempts is the number of corrective generation calls actually
|
||||||
// zero for the public Engine.
|
// started for this result.
|
||||||
RepairAttempts int `json:"repair_attempts"`
|
RepairAttempts int `json:"repair_attempts"`
|
||||||
// IsValid is true for ValidationPassed and ValidationSkipped and false for
|
// IsValid is true for ValidationPassed and ValidationSkipped and false for
|
||||||
// ValidationFailed.
|
// ValidationFailed.
|
||||||
@@ -694,9 +714,8 @@ type GenerateRequest struct {
|
|||||||
// GenerateResponse is returned by an injected LLM client and has a stable JSON
|
// GenerateResponse is returned by an injected LLM client and has a stable JSON
|
||||||
// representation.
|
// representation.
|
||||||
type GenerateResponse struct {
|
type GenerateResponse struct {
|
||||||
// Content is the generated output. It must be non-empty when using the
|
// Content is the generated output. It may be explicitly empty; Promptkit
|
||||||
// built-in client; injected clients may return empty content for Promptkit
|
// applies the effective output contract to classify it.
|
||||||
// validation to classify.
|
|
||||||
Content string `json:"content"`
|
Content string `json:"content"`
|
||||||
// Usage is the client's token accounting.
|
// Usage is the client's token accounting.
|
||||||
Usage TokenUsage `json:"usage"`
|
Usage TokenUsage `json:"usage"`
|
||||||
|
|||||||
Reference in New Issue
Block a user