Compare commits
12 Commits
v0.2.0
...
369ab5392d
| Author | SHA1 | Date | |
|---|---|---|---|
| 369ab5392d | |||
| 2ba0146e5d | |||
| 6112c2af0c | |||
| f5e12c00f5 | |||
| 49fe402dd2 | |||
| c301eb8d55 | |||
| c13e9710d9 | |||
| 87b5ec3d75 | |||
| cb4028a637 | |||
| 5a1bff4529 | |||
| 805a7f965d | |||
| 147f5e5ff5 |
@@ -33,6 +33,9 @@ boundary and constraints that framework work must preserve.
|
|||||||
|
|
||||||
## Release Guidance
|
## Release Guidance
|
||||||
|
|
||||||
|
Consumers upgrading from `v0.2.0` to `v0.3.0` should read the
|
||||||
|
[v0.3.0 changelog](docs/releases/v0.3.0.md).
|
||||||
|
|
||||||
Consumers moving from `v0.1.0` to `v0.2.0` should read the
|
Consumers moving from `v0.1.0` to `v0.2.0` should read the
|
||||||
[v0.2.0 changelog and migration guide](docs/releases/v0.2.0.md).
|
[v0.2.0 changelog and migration guide](docs/releases/v0.2.0.md).
|
||||||
|
|
||||||
|
|||||||
37
backends.go
37
backends.go
@@ -9,6 +9,11 @@ import (
|
|||||||
// backend.
|
// backend.
|
||||||
const BackendOpenRouter = backend.OpenRouterID
|
const BackendOpenRouter = backend.OpenRouterID
|
||||||
|
|
||||||
|
// BackendLocal is the case-sensitive conventional ID used by [LocalBackend].
|
||||||
|
// It is not a built-in or reserved backend and must be registered with
|
||||||
|
// [WithBackend].
|
||||||
|
const BackendLocal = "local"
|
||||||
|
|
||||||
// Backend configures one engine-scoped OpenAI-compatible backend.
|
// Backend configures one engine-scoped OpenAI-compatible backend.
|
||||||
//
|
//
|
||||||
// Backend has no stable JSON representation. Use keyed literals so additions
|
// Backend has no stable JSON representation. Use keyed literals so additions
|
||||||
@@ -35,15 +40,35 @@ type Backend struct {
|
|||||||
// calls allowed for this backend within one Engine. Zero leaves the backend
|
// calls allowed for this backend within one Engine. Zero leaves the backend
|
||||||
// unlimited. A negative value makes NewEngine fail with ErrInvalidConfig.
|
// unlimited. A negative value makes NewEngine fail with ErrInvalidConfig.
|
||||||
ConcurrencyLimit int
|
ConcurrencyLimit int
|
||||||
// QueueCapacity controls how many additional Run calls may be admitted
|
// QueueCapacity controls how many additional Run or RunPrepared calls may
|
||||||
// beyond ConcurrencyLimit. Nil uses 1024 when ConcurrencyLimit is positive;
|
// be admitted beyond ConcurrencyLimit. Nil uses 1024 when ConcurrencyLimit
|
||||||
// a pointer uses its exact value, including zero. The pointed-to value must
|
// is positive; a pointer uses its exact value, including zero. The pointed-to
|
||||||
// be non-negative, and QueueCapacity must be nil when ConcurrencyLimit is
|
// value must be non-negative, and QueueCapacity must be nil when
|
||||||
// zero. Their sum must fit in an int. WithBackend copies the value and does
|
// ConcurrencyLimit is zero. Their sum must fit in an int. WithBackend copies
|
||||||
// not retain the pointer.
|
// the value and does not retain the pointer.
|
||||||
QueueCapacity *int
|
QueueCapacity *int
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// LocalBackend returns a caller-owned Backend for a conventional local
|
||||||
|
// OpenAI-compatible endpoint. It sets ID to BackendLocal and copies endpoint
|
||||||
|
// and concurrencyLimit into Endpoint and ConcurrencyLimit without
|
||||||
|
// normalization or validation. APIKeyEnv, ExtraParams, and QueueCapacity keep
|
||||||
|
// their zero values.
|
||||||
|
//
|
||||||
|
// LocalBackend does not read environment variables, register the value, or
|
||||||
|
// mutate engine or package state. Supply the returned value through
|
||||||
|
// [WithBackend]; [NewEngine] then applies the ordinary backend validation and
|
||||||
|
// concurrency semantics, including default queue capacity for a positive
|
||||||
|
// limit, unlimited behavior for zero, and ErrInvalidConfig for a negative
|
||||||
|
// limit.
|
||||||
|
func LocalBackend(endpoint string, concurrencyLimit int) Backend {
|
||||||
|
return Backend{
|
||||||
|
ID: BackendLocal,
|
||||||
|
Endpoint: endpoint,
|
||||||
|
ConcurrencyLimit: concurrencyLimit,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// WithBackend adds one Backend registration to the constructed Engine.
|
// WithBackend adds one Backend registration to the constructed Engine.
|
||||||
//
|
//
|
||||||
// Registrations accumulate in option order. Every normalized ID must be unique
|
// Registrations accumulate in option order. Every normalized ID must be unique
|
||||||
|
|||||||
37
doc.go
37
doc.go
@@ -3,23 +3,25 @@
|
|||||||
//
|
//
|
||||||
// Applications construct an [Engine] with [NewEngine], select filesystem or
|
// Applications construct an [Engine] with [NewEngine], select filesystem or
|
||||||
// in-memory sources and optional engine-scoped [Backend] registrations, and
|
// in-memory sources and optional engine-scoped [Backend] registrations, and
|
||||||
// call [Engine.Prepare] or [Engine.Run]. Concrete registries, repositories,
|
// call [Engine.Prepare], [Engine.PrepareExecution], [Engine.Run], or
|
||||||
// validators, and the built-in OpenAI-compatible client remain internal
|
// [Engine.RunPrepared]. Concrete registries, repositories, validators, and the
|
||||||
// implementation details.
|
// built-in OpenAI-compatible client remain internal implementation details.
|
||||||
//
|
//
|
||||||
// # Concurrency and ownership
|
// # Concurrency and ownership
|
||||||
//
|
//
|
||||||
// An Engine supports concurrent Prepare and Run calls. Engine-local backend
|
// An Engine supports concurrent Prepare, PrepareExecution, Run, and RunPrepared
|
||||||
// policies bound admitted Run calls and model generations where configured,
|
// calls. Engine-local backend policies bound admitted Run and RunPrepared calls
|
||||||
// while different backend pools and unlimited backends continue independently.
|
// and model generations where configured, while different backend pools and
|
||||||
// An injected [LLMClient] or [ArtifactReader] can therefore still receive
|
// unlimited backends continue independently. An injected [LLMClient] or
|
||||||
// concurrent calls and must be safe for that use.
|
// [ArtifactReader] can therefore still receive concurrent calls and must be
|
||||||
|
// safe for that use.
|
||||||
//
|
//
|
||||||
// NewEngine copies in-memory profiles and backend definitions. Prepare and Run
|
// NewEngine copies in-memory profiles and backend definitions. Prepare,
|
||||||
// copy request maps, slices, pointer values, and JSON-compatible extra
|
// PrepareExecution, and Run copy request maps, slices, pointer values, and
|
||||||
// parameters before using them. Returned values and values passed to extension
|
// JSON-compatible extra parameters before using them. Returned values and
|
||||||
// interfaces are likewise isolated from engine state. Callers own those copies
|
// values passed to extension interfaces are likewise isolated from engine
|
||||||
// and may mutate them after the call that supplied or returned them.
|
// state. Callers own those copies and may mutate them after the call that
|
||||||
|
// supplied or returned them.
|
||||||
//
|
//
|
||||||
// # Security and sensitive data
|
// # Security and sensitive data
|
||||||
//
|
//
|
||||||
@@ -45,10 +47,11 @@
|
|||||||
// [GenerateResponse], [ExecutionTargetPresence], and the string value types
|
// [GenerateResponse], [ExecutionTargetPresence], and the string value types
|
||||||
// used by those values.
|
// used by those values.
|
||||||
//
|
//
|
||||||
// Construction values, including [Config], [Backend], [RunRequest],
|
// Construction and handle values, including [Config], [Backend], [RunRequest],
|
||||||
// [ArtifactRef], [ExecutionTargetOverride], [Profile], and
|
// [ArtifactRef], [ExecutionTargetOverride], [Profile],
|
||||||
// [OpenAICompatibleProfileConfig], do not have stable JSON representations.
|
// [OpenAICompatibleProfileConfig], and [PreparedExecution], do not have stable
|
||||||
// Direct API keys are nevertheless excluded from JSON for every public value.
|
// JSON representations. Direct API keys are nevertheless excluded from JSON
|
||||||
|
// for every public value.
|
||||||
//
|
//
|
||||||
// JSON timestamps use time.Time's RFC 3339 encoding and are omitted when zero.
|
// JSON timestamps use time.Time's RFC 3339 encoding and are omitted when zero.
|
||||||
// PreparedRun and RunResult durations are encoded as integer milliseconds in
|
// PreparedRun and RunResult durations are encoded as integer milliseconds in
|
||||||
|
|||||||
@@ -44,7 +44,9 @@ validation, and profile precedence are defined by the
|
|||||||
|
|
||||||
[`Engine.Prepare`](../../engine.go) resolves the selected prompt and profile,
|
[`Engine.Prepare`](../../engine.go) resolves the selected prompt and profile,
|
||||||
loads inputs and any structured-output schema, and renders messages without
|
loads inputs and any structured-output schema, and renders messages without
|
||||||
calling a model client:
|
calling a model client. Choose it when the prepared value is the final
|
||||||
|
inspection or persistence result and no later execution must be tied to that
|
||||||
|
exact snapshot:
|
||||||
|
|
||||||
```go
|
```go
|
||||||
prepared, err := engine.Prepare(ctx, promptkit.RunRequest{
|
prepared, err := engine.Prepare(ctx, promptkit.RunRequest{
|
||||||
@@ -61,12 +63,48 @@ shows a complete runnable setup with a prompt file, in-memory profile, and
|
|||||||
inline input. Exact request requirements and prepared-result fields belong to
|
inline input. Exact request requirements and prepared-result fields belong to
|
||||||
the [`RunRequest` and `PreparedRun` GoDoc](../../types.go).
|
the [`RunRequest` and `PreparedRun` GoDoc](../../types.go).
|
||||||
|
|
||||||
|
## Prepare Now And Execute The Same Snapshot Later
|
||||||
|
|
||||||
|
Use [`Engine.PrepareExecution`](../../engine.go) when an application must
|
||||||
|
inspect or persist preflight details before deciding whether to start model
|
||||||
|
work, while ensuring that later execution uses those exact rendered messages,
|
||||||
|
inputs, target settings, and validation resources:
|
||||||
|
|
||||||
|
```go
|
||||||
|
preparedExecution, err := engine.PrepareExecution(ctx, promptkit.RunRequest{
|
||||||
|
PromptID: "meeting.summary",
|
||||||
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
|
"note": promptkit.Inline("Synthetic meeting notes"),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer preparedExecution.Discard()
|
||||||
|
|
||||||
|
details := preparedExecution.Details()
|
||||||
|
// Inspect or persist an application-selected safe subset of details.
|
||||||
|
|
||||||
|
result, err := engine.RunPrepared(ctx, preparedExecution)
|
||||||
|
```
|
||||||
|
|
||||||
|
Preparation does not call the model or reserve backend capacity.
|
||||||
|
`RunPrepared` executes from the retained snapshot rather than reloading
|
||||||
|
consumer sources. The handle is opaque in-process state, while `Details`
|
||||||
|
contains rendered content and remains subject to the application's data
|
||||||
|
handling policy. The
|
||||||
|
[`PreparedExecution` and method GoDoc](../../prepared_execution.go) and
|
||||||
|
[engine operation GoDoc](../../engine.go) own exact lifecycle, engine-binding,
|
||||||
|
credential, cancellation, timing, and error semantics.
|
||||||
|
|
||||||
## Execute And Validate
|
## Execute And Validate
|
||||||
|
|
||||||
[`Engine.Run`](../../engine.go) performs the same preparation, invokes the
|
[`Engine.Run`](../../engine.go) performs the same preparation, invokes the
|
||||||
configured model client, classifies the generated artifact, and validates the
|
configured model client, classifies the generated artifact, and validates the
|
||||||
content. A completed content check may return `ValidationFailed` in the result;
|
content in one call. Choose it when the application does not need a preflight
|
||||||
an operational inability to validate returns an error.
|
boundary tied to the eventual execution. A completed content check may return
|
||||||
|
`ValidationFailed` in the result; an operational inability to validate returns
|
||||||
|
an error.
|
||||||
|
|
||||||
The maintained
|
The maintained
|
||||||
[offline execution example](../../examples/go-library/run/main.go) injects a
|
[offline execution example](../../examples/go-library/run/main.go) injects a
|
||||||
@@ -124,35 +162,86 @@ providers. The
|
|||||||
[`RunRequest` and `ExecutionTargetOverride` GoDoc](../../types.go) owns the
|
[`RunRequest` and `ExecutionTargetOverride` GoDoc](../../types.go) owns the
|
||||||
exact normalization, precedence, error, copying, and exposure contract.
|
exact normalization, precedence, error, copying, and exposure contract.
|
||||||
|
|
||||||
### Register A Custom Backend
|
### Configure A Local OpenAI-Compatible Endpoint
|
||||||
|
|
||||||
Register a reusable OpenAI-compatible connection once, then select it from a
|
Choose the smallest configuration that fits how the endpoint will be reused.
|
||||||
profile. This local backend limits model generation to two simultaneous calls;
|
|
||||||
because `QueueCapacity` is omitted, the engine admits up to 1024 additional
|
#### Use An Endpoint-Only Profile
|
||||||
calls waiting behind them:
|
|
||||||
|
Put the endpoint directly on an in-memory profile when only that profile needs
|
||||||
|
it and shared backend identity or capacity policy is unnecessary:
|
||||||
|
|
||||||
```go
|
```go
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
PromptDir: "prompts",
|
PromptDir: "prompts",
|
||||||
},
|
},
|
||||||
promptkit.WithBackend(promptkit.Backend{
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
ID: "local",
|
ID: "local-summary",
|
||||||
Endpoint: "http://localhost:8000/v1",
|
Endpoint: "http://localhost:8000/v1",
|
||||||
APIKeyEnv: "LOCAL_LLM_API_KEY",
|
Model: "example-model",
|
||||||
ConcurrencyLimit: 2,
|
|
||||||
}),
|
}),
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
Endpoint-only profiles have an empty backend ID and remain unrestricted by
|
||||||
|
backend capacity policy.
|
||||||
|
|
||||||
|
#### Use The Conventional Local Backend
|
||||||
|
|
||||||
|
Use `LocalBackend` when profiles should share the conventional `local`
|
||||||
|
identity, endpoint, and concurrency limit:
|
||||||
|
|
||||||
|
```go
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
|
PromptDir: "prompts",
|
||||||
|
},
|
||||||
|
promptkit.WithBackend(
|
||||||
|
promptkit.LocalBackend("http://localhost:8000/v1", 2),
|
||||||
|
),
|
||||||
promptkit.WithProfiles(promptkit.Profile{
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
ID: "local-summary",
|
ID: "local-summary",
|
||||||
BackendID: "local",
|
BackendID: promptkit.BackendLocal,
|
||||||
Model: "example-model",
|
Model: "example-model",
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
```
|
```
|
||||||
|
|
||||||
|
The helper is explicit: it does not pre-register a backend or read environment
|
||||||
|
variables. Supplying a positive limit leaves queue capacity omitted, so normal
|
||||||
|
backend registration selects the existing default waiting capacity of 1024.
|
||||||
|
The returned value still enters the engine through `WithBackend`.
|
||||||
|
|
||||||
|
#### Configure A Complete Backend
|
||||||
|
|
||||||
|
Use a keyed `Backend` value for authentication, extra request parameters, an
|
||||||
|
explicit queue capacity, a custom ID, or multiple local endpoints:
|
||||||
|
|
||||||
|
```go
|
||||||
|
noWaiting := 0
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
|
PromptDir: "prompts",
|
||||||
|
},
|
||||||
|
promptkit.WithBackend(promptkit.Backend{
|
||||||
|
ID: "local-gpu",
|
||||||
|
Endpoint: "http://gpu-host:8000/v1",
|
||||||
|
APIKeyEnv: "LOCAL_GPU_API_KEY",
|
||||||
|
ExtraParams: map[string]any{"provider_option": "enabled"},
|
||||||
|
ConcurrencyLimit: 2,
|
||||||
|
QueueCapacity: &noWaiting,
|
||||||
|
}),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "gpu-summary",
|
||||||
|
BackendID: "local-gpu",
|
||||||
|
Model: "example-model",
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
Use distinct custom IDs when registering multiple local endpoints.
|
||||||
Registrations belong to one engine and custom IDs cannot replace built-ins.
|
Registrations belong to one engine and custom IDs cannot replace built-ins.
|
||||||
The [`Backend` and `WithBackend` GoDoc](../../backends.go) defines validation,
|
The [`Backend`, `LocalBackend`, and `WithBackend` GoDoc](../../backends.go)
|
||||||
copying, uniqueness, exact concurrency-field semantics, and request-default
|
defines exact construction, validation, copying, uniqueness, concurrency, and
|
||||||
behavior.
|
request-default behavior.
|
||||||
|
|
||||||
Both file-backed and in-memory profiles select a registration through
|
Both file-backed and in-memory profiles select a registration through
|
||||||
`backend` or `Profile.BackendID`. Profile and request endpoint overrides retain
|
`backend` or `Profile.BackendID`. Profile and request endpoint overrides retain
|
||||||
@@ -173,8 +262,8 @@ zero:
|
|||||||
```go
|
```go
|
||||||
noWaiting := 0
|
noWaiting := 0
|
||||||
backend := promptkit.Backend{
|
backend := promptkit.Backend{
|
||||||
ID: "local",
|
ID: "local-gpu",
|
||||||
Endpoint: "http://localhost:8000/v1",
|
Endpoint: "http://gpu-host:8000/v1",
|
||||||
ConcurrencyLimit: 2,
|
ConcurrencyLimit: 2,
|
||||||
QueueCapacity: &noWaiting,
|
QueueCapacity: &noWaiting,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -29,21 +29,28 @@ One pool owns immutable active and total limits plus mutex-protected admission
|
|||||||
count, active count, and ordered waiter list. Pool state exists only for the
|
count, active count, and ordered waiter list. Pool state exists only for the
|
||||||
lifetime of its engine.
|
lifetime of its engine.
|
||||||
|
|
||||||
## Bounded Run Admission
|
## Bounded Execution Admission
|
||||||
|
|
||||||
The runner asks the manager to admit a run after resolving the prompt, profile,
|
For ordinary `Run`, the runner asks the manager to admit after resolving the
|
||||||
selected backend, effective execution target, credentials, and output contract,
|
prompt, profile, selected backend, effective execution target, credentials, and
|
||||||
but before schema loading, artifact loading, or rendering. Admission is
|
output contract, but before schema loading, artifact loading, or rendering.
|
||||||
immediate: a limited pool either reserves a slot or returns the internal
|
`PrepareExecution` performs no admission. `RunPrepared` claims its handle,
|
||||||
`ErrCapacityExceeded` identity. The root facade maps that identity to the
|
rechecks credential availability, and then asks the manager to admit the
|
||||||
public error without treating it as an invalid request or generation failure.
|
frozen backend before generation.
|
||||||
|
|
||||||
|
Admission is immediate: a limited pool either reserves a slot or returns the
|
||||||
|
internal `ErrCapacityExceeded` identity. The root facade maps that identity to
|
||||||
|
the public error without treating it as an invalid request or generation
|
||||||
|
failure.
|
||||||
|
|
||||||
The total admitted bound is the active-generation limit plus its configured
|
The total admitted bound is the active-generation limit plus its configured
|
||||||
waiting capacity. The returned release function is idempotent. The runner
|
waiting capacity. The returned release function is idempotent. The runner
|
||||||
defers it as soon as admission succeeds and holds the lease across remaining
|
defers it as soon as admission succeeds. An ordinary run holds the lease across
|
||||||
preparation, initial generation, validation, every repair attempt, and all
|
remaining preparation, initial generation, validation, every repair attempt,
|
||||||
failure or cancellation exits. A repair is part of its original admission and
|
and all failure or cancellation exits. Prepared execution holds the normal
|
||||||
does not reserve another bounded slot.
|
lease across generation, validation, every internal repair attempt, and all
|
||||||
|
execution exits. A repair is part of its original admission and does not
|
||||||
|
reserve another bounded slot.
|
||||||
|
|
||||||
## FIFO Generation Permits
|
## FIFO Generation Permits
|
||||||
|
|
||||||
@@ -90,11 +97,17 @@ unlimited admission. The
|
|||||||
FIFO transfer, canceled-waiter removal, grant/cancel races, independent pools,
|
FIFO transfer, canceled-waiter removal, grant/cancel races, independent pools,
|
||||||
unlimited calls, passthrough behavior, and panic release.
|
unlimited calls, passthrough behavior, and panic release.
|
||||||
|
|
||||||
The [runner tests](../../internal/usecase/runner_test.go) own early admission,
|
The [runner tests](../../internal/usecase/runner_test.go) own ordinary early
|
||||||
lease lifetime, failure release, and shared initial/repair scheduling. The
|
admission, lease lifetime, failure release, and shared initial/repair
|
||||||
|
scheduling. The
|
||||||
|
[prepared-execution use-case tests](../../internal/usecase/prepared_execution_test.go)
|
||||||
|
own deferred admission, credential ordering, and prepared-execution lease
|
||||||
|
release. The
|
||||||
[external package capacity tests](../../capacity_contract_test.go) own the
|
[external package capacity tests](../../capacity_contract_test.go) own the
|
||||||
assembled public-engine behavior for configured limits, capacity errors,
|
assembled public-engine behavior for configured limits, capacity errors,
|
||||||
endpoint identity, engine independence, and injected clients. The
|
endpoint identity, engine independence, and injected clients. The
|
||||||
|
[prepared-execution contract tests](../../prepared_execution_contract_test.go)
|
||||||
|
own the public prepared-capacity boundary. The
|
||||||
[root error-boundary tests](../../errors_internal_test.go) own preservation of
|
[root error-boundary tests](../../errors_internal_test.go) own preservation of
|
||||||
the public generation category and context identity when generation is
|
the public generation category and context identity when generation is
|
||||||
canceled.
|
canceled.
|
||||||
|
|||||||
@@ -43,6 +43,21 @@ without making the model client depend on registry configuration.
|
|||||||
The implementation has no retry loop, tool-call support, provider catalog,
|
The implementation has no retry loop, tool-call support, provider catalog,
|
||||||
inbound HTTP behavior, or durable session store.
|
inbound HTTP behavior, or durable session store.
|
||||||
|
|
||||||
|
## Prepared Generation
|
||||||
|
|
||||||
|
For [`RunPrepared`](../../engine.go), the runner supplies the model client with
|
||||||
|
the target, rendered messages, and structured-output constraint retained by
|
||||||
|
executable preparation. Execution does not reopen or rerender consumer
|
||||||
|
sources.
|
||||||
|
|
||||||
|
Before backend admission, the runner rechecks that the frozen credential
|
||||||
|
environment-variable name is available. The handle does not retain the
|
||||||
|
environment value; the model client resolves the value visible when generation
|
||||||
|
begins. A direct request key remains in private execution state only until the
|
||||||
|
claimed execution finishes or an unclaimed handle is discarded. Exact public
|
||||||
|
ownership and redaction semantics belong to the
|
||||||
|
[`PreparedExecution` GoDoc](../../prepared_execution.go).
|
||||||
|
|
||||||
## Failure Categories
|
## Failure Categories
|
||||||
|
|
||||||
The package preserves distinct error identities for invalid client
|
The package preserves distinct error identities for invalid client
|
||||||
|
|||||||
@@ -11,23 +11,23 @@ contributor workflow and validation.
|
|||||||
|
|
||||||
| Component | Implemented responsibility | References |
|
| Component | Implemented responsibility | References |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request and result values, profile construction, extension interfaces, value conversion, redacted formatting, public error mapping, and engine-local assembly. | [Package GoDoc](../../doc.go), [backend API](../../backends.go), [engine assembly](../../engine.go) |
|
| Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request and result values, opaque prepared-execution handles, profile construction, extension interfaces, value conversion, redacted formatting, public error mapping, and engine-local assembly. | [Package GoDoc](../../doc.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine assembly](../../engine.go) |
|
||||||
| `examples/go-library/prepare` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, and `Prepare`. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
|
| `examples/go-library/prepare` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, and `Prepare`. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
|
||||||
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
||||||
| `internal/backend` | Constructs each engine's immutable registry from the built-in OpenRouter definition and consumer additions, validates and defensively copies definitions through the shared JSON-value package, and consumes the LLM-owned OpenAI-compatible reserved request-field rule. | [Backend registry](../../internal/backend/registry.go) |
|
| `internal/backend` | Constructs each engine's immutable registry from the built-in OpenRouter definition and consumer additions, validates and defensively copies definitions through the shared JSON-value package, and consumes the LLM-owned OpenAI-compatible reserved request-field rule. | [Backend registry](../../internal/backend/registry.go) |
|
||||||
| `internal/capacity` | Owns engine-local bounded run admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
|
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
|
||||||
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation. | [Domain declarations](../../internal/domain/domain.go) |
|
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation. | [Domain declarations](../../internal/domain/domain.go) |
|
||||||
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
||||||
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
||||||
| `internal/jsonvalue` | Validates and deeply copies JSON-compatible extra-parameter trees while preserving supported concrete value types. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
| `internal/jsonvalue` | Validates and deeply copies JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
||||||
| `internal/promptdef` | Loads strictly decoded, validated prompt definitions from filesystem and `fs.FS` sources, including version selection and contained file-backed message content. | [Framework formats](../formats.md), [prompt-definition repository](../../internal/promptdef/filesystem_repository.go) |
|
| `internal/promptdef` | Loads strictly decoded, validated prompt definitions from filesystem and `fs.FS` sources, including version selection and contained file-backed message content. | [Framework formats](../formats.md), [prompt-definition repository](../../internal/promptdef/filesystem_repository.go) |
|
||||||
| `internal/profile` | Loads strictly decoded, validated execution profiles, including backend selection, from filesystem and `fs.FS` sources and composes repositories with error-preserving fallback. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go) |
|
| `internal/profile` | Loads strictly decoded, validated execution profiles, including backend selection, from filesystem and `fs.FS` sources and composes repositories with error-preserving fallback. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go) |
|
||||||
| `internal/profile/builtin` | Embeds the built-in profile catalog, whose entries select OpenRouter, and combines it with an optional primary repository. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) |
|
| `internal/profile/builtin` | Embeds the built-in profile catalog, whose entries select OpenRouter, and combines it with an optional primary repository. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) |
|
||||||
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
|
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
|
||||||
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
||||||
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates frozen validation plans for prepared execution. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
||||||
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
||||||
| `internal/usecase` | Resolves backend, profile, and request settings and coordinates preparation and execution across internal sources, rendering, artifact loading, generation, validation, and optional repair. | [Internal runner](runner.md) |
|
| `internal/usecase` | Resolves backend, profile, and request settings and coordinates preparation, ordinary execution, and one-attempt prepared execution across internal sources, rendering, artifact loading, generation, validation, capacity, and optional repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
|
||||||
|
|
||||||
The root package assembles these internal components without exposing their
|
The root package assembles these internal components without exposing their
|
||||||
representations. Consumers depend only on the root facade.
|
representations. Consumers depend only on the root facade.
|
||||||
|
|||||||
@@ -68,6 +68,22 @@ filesystem or an `fs.FS`. Invalid generated content is returned as a validation
|
|||||||
result; inability to load, register, or compile a schema is an operational
|
result; inability to load, register, or compile a schema is an operational
|
||||||
error.
|
error.
|
||||||
|
|
||||||
|
For executable preparation, the built-in validators create a frozen validation
|
||||||
|
plan. None, basic, and JSON modes retain the effective output contract without
|
||||||
|
source access. JSON Schema mode loads the root document, resolves and compiles
|
||||||
|
every transitive reference during preparation, and retains the compiled
|
||||||
|
validator. The provider-facing structured-output metadata uses that same
|
||||||
|
captured root document.
|
||||||
|
|
||||||
|
`PrepareExecution` also completes prompt and profile selection, artifact
|
||||||
|
loading and hashing, session and message rendering, and target resolution.
|
||||||
|
`RunPrepared` uses the retained source-derived state and validation plan; it
|
||||||
|
does not reopen prompt, profile, input, or schema sources and does not rerender
|
||||||
|
the request. By contrast, ordinary `Prepare` produces an inspection value only:
|
||||||
|
a later `Run` performs its own source resolution and preparation.
|
||||||
|
|
||||||
The [validator tests](../../internal/validate/standard_validator_test.go) own
|
The [validator tests](../../internal/validate/standard_validator_test.go) own
|
||||||
basic, JSON, JSON Schema, source resolution, schema loading, compilation, and
|
basic, JSON, JSON Schema, source resolution, schema loading, compilation,
|
||||||
content-failure behavior.
|
frozen-reference behavior, and content-failure behavior. Prepared execution
|
||||||
|
orchestration is owned by the
|
||||||
|
[use-case tests](../../internal/usecase/prepared_execution_test.go).
|
||||||
|
|||||||
@@ -90,7 +90,7 @@ engine, err := promptkit.NewEngine(
|
|||||||
```
|
```
|
||||||
|
|
||||||
See the
|
See the
|
||||||
[custom-backend consumer guide](../consumers/pkg-promptkit.md#register-a-custom-backend)
|
[local-endpoint consumer guide](../consumers/pkg-promptkit.md#configure-a-local-openai-compatible-endpoint)
|
||||||
for task-oriented usage. The
|
for task-oriented usage. The
|
||||||
[`Backend` and `WithBackend` GoDoc](../../backends.go) owns exact registration,
|
[`Backend` and `WithBackend` GoDoc](../../backends.go) owns exact registration,
|
||||||
validation, copying, defaulting, and uniqueness semantics. The
|
validation, copying, defaulting, and uniqueness semantics. The
|
||||||
|
|||||||
74
docs/releases/v0.3.0.md
Normal file
74
docs/releases/v0.3.0.md
Normal file
@@ -0,0 +1,74 @@
|
|||||||
|
# Promptkit v0.3.0
|
||||||
|
|
||||||
|
This supplemental changelog summarizes the consumer-facing changes from
|
||||||
|
`v0.2.0` to `v0.3.0`. The annotated `v0.3.0` tag is the authoritative release
|
||||||
|
record. Exact current contracts belong to the linked GoDoc and durable
|
||||||
|
documentation.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
`v0.3.0` adds a concise way to register the common local OpenAI-compatible
|
||||||
|
backend configuration:
|
||||||
|
|
||||||
|
- `BackendLocal` provides the conventional, non-reserved backend ID `"local"`;
|
||||||
|
and
|
||||||
|
- `LocalBackend` constructs an ordinary `Backend` from an endpoint and
|
||||||
|
concurrency limit.
|
||||||
|
|
||||||
|
The helper is explicit and additive. It does not pre-register a backend, read
|
||||||
|
environment variables, select a model, or replace the complete `Backend`
|
||||||
|
configuration interface.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
Existing `v0.2.0` consumers require no migration. Endpoint-only profiles,
|
||||||
|
complete custom `Backend` values, the built-in OpenRouter backend, and existing
|
||||||
|
registrations using the literal ID `"local"` continue to work unchanged.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
Update the module dependency with:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get gitea.maximumdirect.net/eric/promptkit@v0.3.0
|
||||||
|
go mod tidy
|
||||||
|
```
|
||||||
|
|
||||||
|
Run the consuming project's ordinary tests and race-enabled tests after the
|
||||||
|
upgrade.
|
||||||
|
|
||||||
|
## Configure A Local Backend
|
||||||
|
|
||||||
|
Register the convenience value through the existing `WithBackend` option and
|
||||||
|
select it from one or more profiles:
|
||||||
|
|
||||||
|
```go
|
||||||
|
engine, err := promptkit.NewEngine(
|
||||||
|
promptkit.Config{PromptDir: "prompts"},
|
||||||
|
promptkit.WithBackend(
|
||||||
|
promptkit.LocalBackend("http://localhost:8000/v1", 2),
|
||||||
|
),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "local-summary",
|
||||||
|
BackendID: promptkit.BackendLocal,
|
||||||
|
Model: "example-model",
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
Use an endpoint-only profile when shared backend identity and capacity policy
|
||||||
|
are unnecessary. Continue to use a complete keyed `Backend` value for custom
|
||||||
|
IDs, authentication, extra request parameters, explicit queue capacity, or
|
||||||
|
multiple local endpoints.
|
||||||
|
|
||||||
|
See the
|
||||||
|
[local-endpoint consumer guide](../consumers/pkg-promptkit.md#configure-a-local-openai-compatible-endpoint)
|
||||||
|
for task-oriented configuration choices. The
|
||||||
|
[`BackendLocal`, `LocalBackend`, and `WithBackend` GoDoc](../../backends.go)
|
||||||
|
owns their exact construction, registration, validation, and concurrency
|
||||||
|
semantics.
|
||||||
|
|
||||||
|
## Consumer Action
|
||||||
|
|
||||||
|
None. Adopt the convenience constructor when it simplifies local endpoint
|
||||||
|
configuration.
|
||||||
@@ -11,7 +11,8 @@ consumer value, and important policy choices.
|
|||||||
This document is planning material, not a description of current behavior.
|
This document is planning material, not a description of current behavior.
|
||||||
Current exported contracts remain owned by Go declarations and GoDoc, backend
|
Current exported contracts remain owned by Go declarations and GoDoc, backend
|
||||||
registration guidance by the
|
registration guidance by the
|
||||||
[consumer guide](../consumers/pkg-promptkit.md#register-a-custom-backend), and
|
[consumer guide](../consumers/pkg-promptkit.md#configure-a-local-openai-compatible-endpoint),
|
||||||
|
and
|
||||||
implemented orchestration by the
|
implemented orchestration by the
|
||||||
[internal runner document](../internal/runner.md).
|
[internal runner document](../internal/runner.md).
|
||||||
|
|
||||||
|
|||||||
@@ -33,9 +33,36 @@ consumers.
|
|||||||
|
|
||||||
## Ideas
|
## Ideas
|
||||||
|
|
||||||
No ideas are currently cataloged. Backend-specific concurrency management has
|
Prompt-independent profile inspection has been selected for active planning in
|
||||||
been selected for active planning in the
|
the [focused feature roadmap](profile-inspection.md). The remaining ideas are
|
||||||
[focused concurrency roadmap](concurrency.md).
|
still available for future selection.
|
||||||
|
|
||||||
|
### Prompt-definition inspection
|
||||||
|
|
||||||
|
Provide exact prompt-definition lookup without rendering, placeholder inputs,
|
||||||
|
profile resolution, or model generation, as requested by
|
||||||
|
[Weatherreporter](weatherreporter-promptkit-wishlist.md#priority-2-prompt-definition-inspection).
|
||||||
|
|
||||||
|
- Return caller-owned identity, version, input definitions, default-profile,
|
||||||
|
output-contract, and opaque definition-equality information.
|
||||||
|
- Apply ordinary prompt-source precedence and exact ID/version selection.
|
||||||
|
- Validate the selected definition and referenced prompt content
|
||||||
|
structurally, without returning source bodies or rendered messages.
|
||||||
|
- Leave complete cross-source corpus validation and enumeration outside the
|
||||||
|
initial inspection contract.
|
||||||
|
|
||||||
|
### Structured capacity errors
|
||||||
|
|
||||||
|
Add safe structured context to backend admission rejection, as requested by
|
||||||
|
[Notarius](notarius-promptkit-wishlist.md#priority-4-structured-capacity-errors)
|
||||||
|
and
|
||||||
|
[Weatherreporter](weatherreporter-promptkit-wishlist.md#structured-capacity-errors).
|
||||||
|
|
||||||
|
- Preserve compatibility with `errors.Is(err, ErrCapacityExceeded)`.
|
||||||
|
- Support `errors.As` to obtain the stable backend ID.
|
||||||
|
- Do not expose endpoints, credential configuration or values, request
|
||||||
|
content, or speculative retry timing.
|
||||||
|
- Keep retry and backoff policy with downstream consumers.
|
||||||
|
|
||||||
## Entry Format
|
## Entry Format
|
||||||
|
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
163
docs/roadmap/local-backend.md
Normal file
163
docs/roadmap/local-backend.md
Normal file
@@ -0,0 +1,163 @@
|
|||||||
|
# Local Backend Convenience
|
||||||
|
|
||||||
|
**Status:** Complete.
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Make the common case of using a local OpenAI-compatible endpoint concise and
|
||||||
|
easy to discover without introducing implicit configuration or a separate
|
||||||
|
backend abstraction.
|
||||||
|
|
||||||
|
The existing `Backend` type and registry remain the canonical, fully
|
||||||
|
configurable interface. A small convenience constructor will cover the usual
|
||||||
|
local-network case, while improved consumer documentation will make it clear
|
||||||
|
when an endpoint-only profile, the convenience constructor, or a complete
|
||||||
|
`Backend` value is appropriate.
|
||||||
|
|
||||||
|
## Motivation
|
||||||
|
|
||||||
|
Consumers can already use a local endpoint by setting `Profile.Endpoint`, or
|
||||||
|
register one as a backend with `WithBackend`. The first option is concise but
|
||||||
|
does not provide shared backend-level concurrency control. The second supports
|
||||||
|
the complete backend feature set but requires consumers to understand and
|
||||||
|
populate several fields for a common configuration.
|
||||||
|
|
||||||
|
Most consumers adding a local backend need only:
|
||||||
|
|
||||||
|
- a stable backend ID;
|
||||||
|
- an OpenAI-compatible endpoint; and
|
||||||
|
- a concurrency limit appropriate for the local server.
|
||||||
|
|
||||||
|
Promptkit should provide a direct path for that case while keeping all
|
||||||
|
configuration explicit and preserving the full registry interface for
|
||||||
|
advanced needs.
|
||||||
|
|
||||||
|
## Consumer Paths
|
||||||
|
|
||||||
|
Documentation should present three progressively more configurable paths:
|
||||||
|
|
||||||
|
1. Set `Profile.Endpoint` when a profile only needs to target a local endpoint
|
||||||
|
and does not need shared backend policy.
|
||||||
|
2. Use the local-backend convenience constructor when profiles should share a
|
||||||
|
named local endpoint and its concurrency limit.
|
||||||
|
3. Construct a complete `Backend` value when the consumer needs a custom
|
||||||
|
backend ID, authentication, extra request parameters, an explicit queue
|
||||||
|
capacity, or multiple local backends.
|
||||||
|
|
||||||
|
These are complementary interfaces. The convenience constructor must return an
|
||||||
|
ordinary `Backend`, so it does not create a second configuration model.
|
||||||
|
|
||||||
|
## Public Convenience API
|
||||||
|
|
||||||
|
The public package should expose:
|
||||||
|
|
||||||
|
```go
|
||||||
|
const BackendLocal = "local"
|
||||||
|
|
||||||
|
func LocalBackend(endpoint string, concurrencyLimit int) Backend
|
||||||
|
```
|
||||||
|
|
||||||
|
`LocalBackend` should return a `Backend` with:
|
||||||
|
|
||||||
|
- `ID` set to `BackendLocal`;
|
||||||
|
- `Endpoint` set to the supplied endpoint;
|
||||||
|
- `ConcurrencyLimit` set to the supplied limit; and
|
||||||
|
- all other fields left at their zero values.
|
||||||
|
|
||||||
|
The returned value is passed to `WithBackend` and follows the same copying,
|
||||||
|
normalization, validation, and registration rules as any consumer-constructed
|
||||||
|
`Backend`.
|
||||||
|
|
||||||
|
The constructor should be a transparent value constructor. It should not read
|
||||||
|
environment variables, mutate global state, register the backend, validate
|
||||||
|
arguments independently, or create profiles. Consumers may inspect or modify
|
||||||
|
the returned value before registration, although documentation should direct
|
||||||
|
substantially customized configurations to the full `Backend` form.
|
||||||
|
|
||||||
|
## Identity and Registration
|
||||||
|
|
||||||
|
`BackendLocal` is a conventional ID used by the convenience constructor. It is
|
||||||
|
not pre-registered and should not become a specially reserved registry ID.
|
||||||
|
Consumers remain responsible for registering the returned backend with
|
||||||
|
`WithBackend` and naming it from profiles through `BackendID`.
|
||||||
|
|
||||||
|
This distinction preserves compatibility with consumers that may already
|
||||||
|
register their own backend using the ID `"local"`. Normal duplicate-ID rules
|
||||||
|
still apply if a consumer attempts to register more than one backend with that
|
||||||
|
ID.
|
||||||
|
|
||||||
|
Consumers that need multiple local endpoints should choose distinct IDs and
|
||||||
|
use complete `Backend` values rather than the single conventional helper ID.
|
||||||
|
|
||||||
|
## Concurrency and Queue Semantics
|
||||||
|
|
||||||
|
The constructor must preserve the existing backend concurrency contract:
|
||||||
|
|
||||||
|
- a positive concurrency limit bounds simultaneous requests and uses the
|
||||||
|
existing default queue capacity because `QueueCapacity` remains `nil`;
|
||||||
|
- a zero concurrency limit leaves the backend unconstrained; and
|
||||||
|
- a negative concurrency limit is rejected through the existing engine
|
||||||
|
configuration validation path.
|
||||||
|
|
||||||
|
The constructor should not select a hidden default concurrency limit. Local
|
||||||
|
servers vary substantially in capacity, so the consumer should make this
|
||||||
|
choice explicitly.
|
||||||
|
|
||||||
|
## Documentation
|
||||||
|
|
||||||
|
The final documentation state has two canonical surfaces:
|
||||||
|
|
||||||
|
- Public Go documentation describes the exact contract of
|
||||||
|
`BackendLocal` and `LocalBackend`, including their conventional,
|
||||||
|
non-pre-registered nature.
|
||||||
|
- The [promptkit consumer guide](../consumers/pkg-promptkit.md) includes
|
||||||
|
a task-oriented local-endpoint section that shows the three consumer paths,
|
||||||
|
explains the decision between them, and provides concise examples of the
|
||||||
|
endpoint-only and convenience-constructor forms.
|
||||||
|
|
||||||
|
The consumer guide continues to document the full `Backend` interface as
|
||||||
|
the advanced path rather than attempting to reproduce every configuration
|
||||||
|
variation through convenience APIs.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
This feature is additive:
|
||||||
|
|
||||||
|
- existing endpoint-only profiles continue to work unchanged;
|
||||||
|
- existing `Backend` values and `WithBackend` registrations remain the
|
||||||
|
canonical general-purpose interface;
|
||||||
|
- existing registrations using the literal ID `"local"` remain valid; and
|
||||||
|
- OpenRouter defaults and all other backend behavior remain unchanged.
|
||||||
|
|
||||||
|
No consumer is required to adopt the convenience constructor.
|
||||||
|
|
||||||
|
## Non-Goals
|
||||||
|
|
||||||
|
This work does not include:
|
||||||
|
|
||||||
|
- pre-registering or implicitly enabling a local backend;
|
||||||
|
- discovering a local endpoint, API key, or concurrency limit from environment
|
||||||
|
variables;
|
||||||
|
- adding local-backend fields to `Config`;
|
||||||
|
- selecting a default local model or creating a profile automatically;
|
||||||
|
- adding a combined backend-and-profile constructor;
|
||||||
|
- adding convenience parameters for API keys, extra request parameters, or
|
||||||
|
queue capacity;
|
||||||
|
- replacing or redesigning the backend registry;
|
||||||
|
- adding support for non-OpenAI-compatible local APIs; or
|
||||||
|
- changing backend routing, scheduling, or queue behavior.
|
||||||
|
|
||||||
|
## Target End State
|
||||||
|
|
||||||
|
After this work:
|
||||||
|
|
||||||
|
- consumers with a simple one-profile local endpoint can continue to configure
|
||||||
|
it directly on the profile;
|
||||||
|
- consumers needing a shared local endpoint and concurrency policy can express
|
||||||
|
it with one `LocalBackend` call and register the returned value normally;
|
||||||
|
- consumers with advanced or multiple-local-backend requirements have a clear
|
||||||
|
path to the complete `Backend` interface;
|
||||||
|
- all local configuration remains explicit, inspectable, and compatible with
|
||||||
|
dependency injection; and
|
||||||
|
- canonical documentation makes the simplest suitable interface easy to find
|
||||||
|
without obscuring the underlying registry model.
|
||||||
358
docs/roadmap/notarius-promptkit-wishlist.md
Normal file
358
docs/roadmap/notarius-promptkit-wishlist.md
Normal file
@@ -0,0 +1,358 @@
|
|||||||
|
# Notarius PromptKit Wishlist
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
This document records features and interface changes that would be useful
|
||||||
|
additions to PromptKit from the perspective of the maintainers of Notarius, a
|
||||||
|
downstream application that consumes PromptKit.
|
||||||
|
|
||||||
|
PromptKit v0.3.0 provides the capabilities Notarius currently needs. None of
|
||||||
|
the ideas below blocks current Notarius development. They are opportunities to
|
||||||
|
reduce downstream workarounds, improve integration correctness, and make
|
||||||
|
PromptKit more ergonomic for applications with configuration validation,
|
||||||
|
debugging, checkpointing, and operational-observability requirements.
|
||||||
|
|
||||||
|
The examples are API sketches intended to communicate the desired capability,
|
||||||
|
not prescriptive names or finalized Go contracts.
|
||||||
|
|
||||||
|
## Priority 1: Atomic Execution With Prepared Details
|
||||||
|
|
||||||
|
**Disposition:** Covered by the accepted
|
||||||
|
[executable preparation handles](prepared-execution.md) roadmap. The shared
|
||||||
|
two-phase capability should provide the required single-preparation
|
||||||
|
consistency; a separate `RunDetailed` method is not cataloged initially.
|
||||||
|
|
||||||
|
### Downstream need
|
||||||
|
|
||||||
|
Notarius needs both:
|
||||||
|
|
||||||
|
- the completed `RunResult`; and
|
||||||
|
- the rendered messages, effective output contract, hashes, and other
|
||||||
|
preparation details exposed by `PreparedRun`.
|
||||||
|
|
||||||
|
Notarius uses the prepared details to construct redaction-aware debug bundles
|
||||||
|
and retain enough information to diagnose model behavior.
|
||||||
|
|
||||||
|
### Current integration
|
||||||
|
|
||||||
|
Notarius currently calls `Engine.Prepare` and then `Engine.Run` with the same
|
||||||
|
request. Because `Run` performs preparation internally, a successful request
|
||||||
|
resolves and prepares the same work twice.
|
||||||
|
|
||||||
|
This duplicates profile resolution, input hashing, schema loading, and prompt
|
||||||
|
rendering. It also creates a theoretical consistency window in which a
|
||||||
|
filesystem-backed prompt, profile, schema, or input could change between the
|
||||||
|
explicit preparation and the preparation performed by `Run`.
|
||||||
|
|
||||||
|
### Requested capability
|
||||||
|
|
||||||
|
Add an opt-in execution method that prepares exactly once and returns both the
|
||||||
|
prepared details and completed result:
|
||||||
|
|
||||||
|
```go
|
||||||
|
type RunReport struct {
|
||||||
|
Prepared PreparedRun
|
||||||
|
Result RunResult
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) RunDetailed(
|
||||||
|
ctx context.Context,
|
||||||
|
req RunRequest,
|
||||||
|
) (*RunReport, error)
|
||||||
|
```
|
||||||
|
|
||||||
|
The exact names are flexible. The important contract is that preparation
|
||||||
|
occurs once and that the returned prepared state describes the execution that
|
||||||
|
produced the returned result.
|
||||||
|
|
||||||
|
Existing `Prepare` and `Run` behavior should remain available for consumers
|
||||||
|
that need only one side of the operation.
|
||||||
|
|
||||||
|
### Design considerations
|
||||||
|
|
||||||
|
- Keep this API additive and preserve the existing simple `Run` workflow.
|
||||||
|
- Return caller-owned copies under PromptKit's existing ownership rules.
|
||||||
|
- Define whether any prepared details are available after an operational
|
||||||
|
generation or validation error. Notarius does not require partial results
|
||||||
|
for the initial use case, but an explicit contract would be valuable.
|
||||||
|
- Preserve cancellation and backend-admission semantics.
|
||||||
|
- Do not add all prepared content directly to `RunResult`. Rendered prompt
|
||||||
|
content can be large and sensitive, and consumers should opt in to receiving
|
||||||
|
it.
|
||||||
|
|
||||||
|
### Value to Notarius
|
||||||
|
|
||||||
|
This is the highest-value wishlist item. It would remove duplicate work from
|
||||||
|
every successful PromptKit-backed call and ensure that retained debug material
|
||||||
|
corresponds atomically to the actual execution.
|
||||||
|
|
||||||
|
## Priority 2: Prompt-Independent Profile Inspection
|
||||||
|
|
||||||
|
**Disposition:** Covered by the accepted
|
||||||
|
[prompt-independent profile inspection](profile-inspection.md) roadmap.
|
||||||
|
|
||||||
|
### Downstream need
|
||||||
|
|
||||||
|
Notarius validates configured pipeline profile IDs before beginning a run. It
|
||||||
|
needs to determine whether:
|
||||||
|
|
||||||
|
- a profile exists;
|
||||||
|
- its referenced backend is registered;
|
||||||
|
- its execution target can be resolved; and
|
||||||
|
- it declares a credential requirement that the application may need to
|
||||||
|
enforce.
|
||||||
|
|
||||||
|
This validation should not require model generation.
|
||||||
|
|
||||||
|
### Current integration
|
||||||
|
|
||||||
|
Notarius constructs a synthetic prompt using `testing/fstest.MapFS`, supplies a
|
||||||
|
dummy transcript, and calls `Engine.Prepare` solely to exercise profile and
|
||||||
|
backend resolution. This works, but prompt preparation is serving as a
|
||||||
|
substitute for a profile-inspection interface.
|
||||||
|
|
||||||
|
### Requested capability
|
||||||
|
|
||||||
|
Add a prompt-independent profile-resolution API, for example:
|
||||||
|
|
||||||
|
```go
|
||||||
|
type ResolvedProfile struct {
|
||||||
|
ProfileID string
|
||||||
|
BackendID string
|
||||||
|
EffectiveTarget ExecutionTarget
|
||||||
|
APIKeyEnv string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) ResolveProfile(
|
||||||
|
ctx context.Context,
|
||||||
|
profileID string,
|
||||||
|
) (ResolvedProfile, error)
|
||||||
|
```
|
||||||
|
|
||||||
|
The returned shape may differ, but it should provide enough information for a
|
||||||
|
consumer to validate an explicit profile selection without inventing a prompt
|
||||||
|
or supplying placeholder inputs.
|
||||||
|
|
||||||
|
### Design considerations
|
||||||
|
|
||||||
|
- Resolve built-in, file-backed, and programmatic profiles using normal
|
||||||
|
PromptKit precedence.
|
||||||
|
- Validate that a referenced backend registration exists.
|
||||||
|
- Do not resolve, retain, or expose credential values.
|
||||||
|
- Report credential requirements, such as an environment-variable name, so
|
||||||
|
the consuming application can decide whether availability is required at
|
||||||
|
configuration-validation time or only at execution time.
|
||||||
|
- Return caller-owned values.
|
||||||
|
- Preserve typed or sentinel error classification for missing and invalid
|
||||||
|
profiles.
|
||||||
|
- Consider accepting an `ExecutionTargetOverride` if consumers need to inspect
|
||||||
|
the same effective target that a run-level override would produce.
|
||||||
|
- Enumeration of all profiles is not required for the Notarius use case; exact
|
||||||
|
lookup by ID is sufficient.
|
||||||
|
|
||||||
|
### Value to Notarius
|
||||||
|
|
||||||
|
This would eliminate a synthetic production-only prompt fixture and establish
|
||||||
|
a direct, supported contract for configuration-time profile and backend
|
||||||
|
validation.
|
||||||
|
|
||||||
|
## Priority 3: Semantic Execution-Target Fingerprints
|
||||||
|
|
||||||
|
**Disposition:** Deferred until prompt-independent profile inspection defines
|
||||||
|
the resolved target whose configuration identity would be fingerprinted.
|
||||||
|
|
||||||
|
### Downstream need
|
||||||
|
|
||||||
|
Notarius checkpoints model-backed pipeline stages. A checkpoint must not be
|
||||||
|
reused when generation-affecting PromptKit configuration changes.
|
||||||
|
|
||||||
|
Notarius therefore needs a stable equality signal for the effective profile
|
||||||
|
and backend target used by a pipeline.
|
||||||
|
|
||||||
|
### Current integration
|
||||||
|
|
||||||
|
Notarius currently constructs this identity itself from:
|
||||||
|
|
||||||
|
- a manually maintained marker for the PromptKit release and built-in profile
|
||||||
|
catalog;
|
||||||
|
- raw hashes of configured profile files; and
|
||||||
|
- a separate hash of the configured conventional local-backend endpoint.
|
||||||
|
|
||||||
|
This is safe but conservative and coupled to PromptKit details. Raw file
|
||||||
|
hashing also invalidates checkpoints for semantically irrelevant YAML changes,
|
||||||
|
such as comments or formatting.
|
||||||
|
|
||||||
|
### Requested capability
|
||||||
|
|
||||||
|
Expose an opaque semantic digest for a resolved profile and its effective
|
||||||
|
generation target. It could be returned by the proposed profile-resolution
|
||||||
|
API:
|
||||||
|
|
||||||
|
```go
|
||||||
|
type ResolvedProfile struct {
|
||||||
|
ProfileID string
|
||||||
|
BackendID string
|
||||||
|
EffectiveTarget ExecutionTarget
|
||||||
|
ExecutionDigest string
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Alternatively, PromptKit could expose a dedicated method such as
|
||||||
|
`ProfileExecutionDigest(profileID)`.
|
||||||
|
|
||||||
|
### Desired equality semantics
|
||||||
|
|
||||||
|
The digest should change when generation-affecting state changes, including:
|
||||||
|
|
||||||
|
- resolved model and endpoint;
|
||||||
|
- backend routing identity;
|
||||||
|
- backend request defaults and extra parameters;
|
||||||
|
- profile generation parameters; and
|
||||||
|
- the semantic identity of any selected built-in profile.
|
||||||
|
|
||||||
|
The digest should not incorporate:
|
||||||
|
|
||||||
|
- credential values;
|
||||||
|
- concurrency or queue capacity;
|
||||||
|
- filesystem source paths;
|
||||||
|
- YAML comments or formatting; or
|
||||||
|
- other settings that affect scheduling or source representation without
|
||||||
|
changing the generation target.
|
||||||
|
|
||||||
|
The credential environment-variable name may need to participate if changing
|
||||||
|
it can select a materially different provider account or target. PromptKit
|
||||||
|
should define this deliberately while continuing to exclude the resolved
|
||||||
|
secret value.
|
||||||
|
|
||||||
|
### Design considerations
|
||||||
|
|
||||||
|
- Treat the digest as an opaque equality value rather than a public encoding
|
||||||
|
of internal structures.
|
||||||
|
- Document which categories of change affect equality.
|
||||||
|
- Include a versioned semantic marker internally so PromptKit can deliberately
|
||||||
|
invalidate old digests when its resolution semantics change.
|
||||||
|
- Prefer a per-profile digest over a digest of every profile known to an
|
||||||
|
engine. Notarius generally knows which profiles a resolved pipeline uses.
|
||||||
|
- Do not require consumers to know PromptKit's built-in catalog version.
|
||||||
|
|
||||||
|
### Value to Notarius
|
||||||
|
|
||||||
|
This would let Notarius remove its PromptKit release marker and raw
|
||||||
|
profile-source fingerprinting, reduce unnecessary checkpoint invalidation, and
|
||||||
|
delegate execution-target equality to the component that owns target
|
||||||
|
resolution.
|
||||||
|
|
||||||
|
## Priority 4: Structured Capacity Errors
|
||||||
|
|
||||||
|
**Disposition:** Accepted into the
|
||||||
|
[future catalog](future.md#structured-capacity-errors).
|
||||||
|
|
||||||
|
### Downstream need
|
||||||
|
|
||||||
|
Notarius translates PromptKit backend-capacity rejection into a
|
||||||
|
provider-neutral application error. When multiple backends are active,
|
||||||
|
operators would benefit from knowing which backend rejected admission without
|
||||||
|
parsing an error string or exposing endpoint details.
|
||||||
|
|
||||||
|
### Current integration
|
||||||
|
|
||||||
|
PromptKit provides the useful `ErrCapacityExceeded` sentinel. Notarius can
|
||||||
|
classify the failure reliably, but it retains only a sanitized diagnostic
|
||||||
|
string as additional context.
|
||||||
|
|
||||||
|
### Requested capability
|
||||||
|
|
||||||
|
Add a typed error that continues to match `ErrCapacityExceeded`:
|
||||||
|
|
||||||
|
```go
|
||||||
|
type CapacityError struct {
|
||||||
|
BackendID string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *CapacityError) Is(target error) bool {
|
||||||
|
return target == ErrCapacityExceeded
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The exact implementation may use `Unwrap` or another idiomatic mechanism. The
|
||||||
|
important properties are compatibility with `errors.Is` and discoverability
|
||||||
|
through `errors.As`.
|
||||||
|
|
||||||
|
### Design considerations
|
||||||
|
|
||||||
|
- Include the stable backend ID.
|
||||||
|
- Do not expose the backend endpoint, credential environment, credential
|
||||||
|
value, request content, or other sensitive configuration.
|
||||||
|
- Consider including the configured concurrency and queue limits if they are
|
||||||
|
useful and safe, but backend identity alone provides most of the downstream
|
||||||
|
value.
|
||||||
|
- Add a retry delay only if PromptKit can provide a meaningful value. A full
|
||||||
|
queue does not necessarily imply a reliable `Retry-After` duration.
|
||||||
|
- Keep retry and backoff policy with the consuming application. PromptKit
|
||||||
|
should classify the admission failure rather than silently retry it.
|
||||||
|
|
||||||
|
### Value to Notarius
|
||||||
|
|
||||||
|
This would improve operational diagnostics and future metrics while preserving
|
||||||
|
the provider-neutral error boundary used by Notarius.
|
||||||
|
|
||||||
|
## Capabilities PromptKit Already Provides Well
|
||||||
|
|
||||||
|
The current PromptKit boundary is sufficient for Notarius's implemented
|
||||||
|
behavior. In particular, PromptKit already provides:
|
||||||
|
|
||||||
|
- filesystem, `fs.FS`, and programmatic prompt, profile, and schema sources;
|
||||||
|
- offline preparation without model execution;
|
||||||
|
- structured output and content validation;
|
||||||
|
- direct session propagation;
|
||||||
|
- tri-state per-run reasoning overrides;
|
||||||
|
- selected profile, backend, model, endpoint, effective parameters, hashes, and
|
||||||
|
token-usage provenance;
|
||||||
|
- endpoint-only profiles;
|
||||||
|
- the conventional `local` backend helper;
|
||||||
|
- arbitrary engine-scoped `Backend` registrations;
|
||||||
|
- backend authentication environment names, extra parameters, concurrency
|
||||||
|
limits, and queue-capacity policies;
|
||||||
|
- provider-client and artifact-reader extension interfaces;
|
||||||
|
- context cancellation; and
|
||||||
|
- useful public error sentinels, including profile absence and capacity
|
||||||
|
exhaustion.
|
||||||
|
|
||||||
|
The wishlist does not imply that Notarius needs PromptKit to broaden its core
|
||||||
|
responsibilities. It primarily asks for more direct access to information and
|
||||||
|
operations that PromptKit already computes internally.
|
||||||
|
|
||||||
|
## Responsibilities That Should Remain In Notarius
|
||||||
|
|
||||||
|
The following concerns belong to the downstream application and should not
|
||||||
|
move into PromptKit for the sake of Notarius:
|
||||||
|
|
||||||
|
- pipeline staging, dependencies, and generated references;
|
||||||
|
- application-wide scheduling across providers and backends;
|
||||||
|
- module and validation retry policy;
|
||||||
|
- checkpoints, resume, and recomputation;
|
||||||
|
- durable run artifacts and manifests;
|
||||||
|
- D&D prompts, schemas, extractors, validators, and normalizers;
|
||||||
|
- Notarius configuration-file parsing and precedence;
|
||||||
|
- domain-specific prompt-cache prefix policy; and
|
||||||
|
- application-specific redaction, retention, and debug-bundle policy.
|
||||||
|
|
||||||
|
PromptKit's complete `Backend` API already supports custom IDs, multiple local
|
||||||
|
endpoints, authentication, extra parameters, and explicit queue policies.
|
||||||
|
Whether Notarius exposes those capabilities in its own configuration is an
|
||||||
|
application-policy decision, not an upstream PromptKit gap.
|
||||||
|
|
||||||
|
## Suggested Upstream Sequence
|
||||||
|
|
||||||
|
If the PromptKit team chooses to pursue these ideas, the most useful order for
|
||||||
|
Notarius would be:
|
||||||
|
|
||||||
|
1. Add atomic execution that returns prepared details and the completed result.
|
||||||
|
2. Add prompt-independent profile inspection.
|
||||||
|
3. Add a semantic execution-target digest, preferably as part of profile
|
||||||
|
inspection.
|
||||||
|
4. Add a typed capacity error carrying backend identity.
|
||||||
|
|
||||||
|
The first two address concrete workarounds in current Notarius code. The third
|
||||||
|
would improve checkpoint correctness and reduce coupling. The fourth is
|
||||||
|
operational polish.
|
||||||
280
docs/roadmap/prepared-execution.md
Normal file
280
docs/roadmap/prepared-execution.md
Normal file
@@ -0,0 +1,280 @@
|
|||||||
|
# Executable Preparation Handles
|
||||||
|
|
||||||
|
**Status:** Complete.
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Allow a consumer to prepare one exact Promptkit execution, inspect and retain
|
||||||
|
its credential-redacted public preparation details, and later execute that
|
||||||
|
already-prepared work without resolving or rendering the request again.
|
||||||
|
|
||||||
|
This provides a supported preflight-before-generation boundary for
|
||||||
|
[Weatherreporter](weatherreporter-promptkit-wishlist.md#priority-1-executable-preparation-handles)
|
||||||
|
and removes the duplicate `Prepare`-then-`Run` workaround described by
|
||||||
|
[Notarius](notarius-promptkit-wishlist.md#priority-1-atomic-execution-with-prepared-details).
|
||||||
|
|
||||||
|
## Motivation
|
||||||
|
|
||||||
|
`Engine.Prepare` currently returns the provenance and rendered details that
|
||||||
|
consumers need for debugging, persistence, and preflight checks. `Engine.Run`
|
||||||
|
then performs its own preparation before generation. A consumer that needs
|
||||||
|
both values must therefore prepare the same logical request twice.
|
||||||
|
|
||||||
|
That workaround duplicates source loading, input hashing, schema work, and
|
||||||
|
template rendering. It also permits filesystem-backed prompts, profiles,
|
||||||
|
schemas, or inputs to change between the public preparation and the
|
||||||
|
preparation that actually produces the result.
|
||||||
|
|
||||||
|
Promptkit already owns the complete preparation and execution pipeline. An
|
||||||
|
opt-in prepared-execution handle should expose the missing boundary without
|
||||||
|
putting rendered content into every `RunResult` or moving persistence policy
|
||||||
|
into the library.
|
||||||
|
|
||||||
|
## Consumer Workflow
|
||||||
|
|
||||||
|
The target public workflow is:
|
||||||
|
|
||||||
|
```go
|
||||||
|
prepared, err := engine.PrepareExecution(ctx, request)
|
||||||
|
if err != nil {
|
||||||
|
// Handle preparation failure.
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer prepared.Discard()
|
||||||
|
|
||||||
|
details := prepared.Details()
|
||||||
|
// Persist or inspect a consumer-selected safe subset of details.
|
||||||
|
|
||||||
|
result, err := engine.RunPrepared(ctx, prepared)
|
||||||
|
```
|
||||||
|
|
||||||
|
The target public surface is:
|
||||||
|
|
||||||
|
```go
|
||||||
|
type PreparedExecution struct {
|
||||||
|
// Opaque Promptkit-owned state.
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) PrepareExecution(
|
||||||
|
ctx context.Context,
|
||||||
|
req RunRequest,
|
||||||
|
) (*PreparedExecution, error)
|
||||||
|
|
||||||
|
func (p *PreparedExecution) Details() PreparedRun
|
||||||
|
|
||||||
|
func (p *PreparedExecution) Discard()
|
||||||
|
|
||||||
|
func (e *Engine) RunPrepared(
|
||||||
|
ctx context.Context,
|
||||||
|
prepared *PreparedExecution,
|
||||||
|
) (*RunResult, error)
|
||||||
|
```
|
||||||
|
|
||||||
|
The declarations and GoDoc will own the exact implemented contract. The
|
||||||
|
important public shape is an opaque handle, caller-owned `PreparedRun`
|
||||||
|
details, an explicit discard operation, and execution through the engine that
|
||||||
|
created the handle.
|
||||||
|
|
||||||
|
## Prepared Snapshot
|
||||||
|
|
||||||
|
`PrepareExecution` performs complete preparation without model generation or
|
||||||
|
backend-capacity admission. It applies the same request validation, source
|
||||||
|
precedence, backend and profile resolution, credential requirement checks,
|
||||||
|
output-contract resolution, artifact loading, hashing, schema loading,
|
||||||
|
session resolution, and rendering behavior as `Prepare`.
|
||||||
|
|
||||||
|
A successful handle freezes all source-derived state required for later
|
||||||
|
execution, including:
|
||||||
|
|
||||||
|
- the selected prompt definition, profile, and backend identity;
|
||||||
|
- the complete effective execution target and request-field presence;
|
||||||
|
- rendered messages and effective session ID;
|
||||||
|
- prompt, rendered-prompt, and input hashes;
|
||||||
|
- the effective output contract and provider-facing structured-output
|
||||||
|
constraint; and
|
||||||
|
- private validation state sufficient to validate generated output without
|
||||||
|
reopening schema files or `fs.FS` resources, including required schema
|
||||||
|
references.
|
||||||
|
|
||||||
|
After `PrepareExecution` succeeds, changes to prompt, profile, schema, input,
|
||||||
|
or request-owned data cannot change what `RunPrepared` sends to the model or
|
||||||
|
how it validates the generated output.
|
||||||
|
|
||||||
|
The handle retains an internal snapshot independent from values returned by
|
||||||
|
`Details`. Mutating a returned `PreparedRun`, its maps, slices, messages, or
|
||||||
|
schema values does not affect later execution. Each `Details` call returns a
|
||||||
|
fresh caller-owned copy under the existing `PreparedRun` ownership and stable
|
||||||
|
JSON rules.
|
||||||
|
|
||||||
|
## Handle Lifecycle
|
||||||
|
|
||||||
|
A `PreparedExecution` is:
|
||||||
|
|
||||||
|
- created only by a successful `PrepareExecution` call;
|
||||||
|
- bound to the exact `Engine` that created it;
|
||||||
|
- valid for one `RunPrepared` invocation;
|
||||||
|
- safe for repeated `Details` calls;
|
||||||
|
- intentionally opaque and without a supported JSON representation; and
|
||||||
|
- in-process state rather than a durable or restartable job.
|
||||||
|
|
||||||
|
`RunPrepared` atomically claims a valid handle before beginning the execution
|
||||||
|
attempt. A second or concurrent invocation fails without starting another
|
||||||
|
execution, including when the first invocation ended in cancellation, capacity
|
||||||
|
rejection, generation failure, or operational validation failure. Copies of
|
||||||
|
the public handle share the same one-attempt state and cannot bypass this rule.
|
||||||
|
|
||||||
|
A nil, zero-value, foreign-engine, discarded, already-claimed, or already-used
|
||||||
|
handle is invalid. `RunPrepared` reports these lifecycle errors through the
|
||||||
|
ordinary public invalid-request category. A failed foreign-engine invocation
|
||||||
|
does not consume a handle that remains valid for its owning engine.
|
||||||
|
|
||||||
|
`Discard` idempotently makes an unclaimed handle unavailable for execution and
|
||||||
|
drops Promptkit's references to secret-bearing or execution-only state.
|
||||||
|
`RunPrepared` performs the same cleanup automatically after claiming a handle.
|
||||||
|
Credential-redacted public preparation details remain available after discard,
|
||||||
|
success, or failure so consumers can retain diagnostics. Promptkit does not
|
||||||
|
promise secure erasure of Go string memory.
|
||||||
|
|
||||||
|
Lifecycle transitions are concurrency-safe. When `RunPrepared` and `Discard`
|
||||||
|
race, exactly one claims the ready handle. `Discard` is not an execution
|
||||||
|
cancellation mechanism and does not interrupt an attempt that has already
|
||||||
|
claimed the handle; consumers cancel that attempt through its context.
|
||||||
|
|
||||||
|
## Credentials And Sensitive Data
|
||||||
|
|
||||||
|
`Details` has the same security contract as `PreparedRun`: it can contain
|
||||||
|
rendered messages, schemas, identifiers, and hashes, but never a resolved API
|
||||||
|
key value. Consumers remain responsible for selecting, redacting, storing, and
|
||||||
|
retaining any persisted preparation material.
|
||||||
|
|
||||||
|
A direct `RunRequest.APIKey` is retained only in opaque execution state until
|
||||||
|
the handle is run or discarded. It is never added to details, hashes, JSON,
|
||||||
|
`String`, or `GoString` output.
|
||||||
|
|
||||||
|
An environment-variable name is frozen as part of the effective target, but
|
||||||
|
its credential value is not captured for the lifetime of the handle.
|
||||||
|
`PrepareExecution` applies the existing preparation-time availability check.
|
||||||
|
`RunPrepared` rechecks availability before admission, and the selected model
|
||||||
|
client uses the environment value visible during execution. This preserves
|
||||||
|
current secret ownership and avoids retaining an environment credential while
|
||||||
|
a consumer persists preflight material.
|
||||||
|
|
||||||
|
The opaque handle must not expose retained request data or credentials through
|
||||||
|
default formatting, JSON, or error messages.
|
||||||
|
|
||||||
|
## Admission, Cancellation, And Execution
|
||||||
|
|
||||||
|
`PrepareExecution` never reserves backend admission or an active-generation
|
||||||
|
permit. Its context governs preparation only; cancellation after it returns
|
||||||
|
does not invalidate the handle.
|
||||||
|
|
||||||
|
`RunPrepared` uses its own context for credential revalidation, backend
|
||||||
|
admission, active-generation waiting, model generation, output validation, and
|
||||||
|
any internal repair. Admission occurs when `RunPrepared` begins so a consumer
|
||||||
|
cannot occupy bounded capacity while inspecting or persisting preparation
|
||||||
|
details.
|
||||||
|
|
||||||
|
For a limited backend, the admission lease covers the complete prepared
|
||||||
|
execution attempt after admission: generation, validation, internal repair,
|
||||||
|
and every success or failure exit. Actual generation continues to use the
|
||||||
|
backend's FIFO active-generation permit. Existing capacity error identity,
|
||||||
|
cancellation behavior, and release guarantees remain in force.
|
||||||
|
|
||||||
|
Because a prepared handle is one-attempt, cancellation or capacity rejection
|
||||||
|
does not make it reusable. Retry and backoff policy remains with the consumer,
|
||||||
|
which may create a new prepared handle when another attempt is appropriate.
|
||||||
|
|
||||||
|
## Results And Failures
|
||||||
|
|
||||||
|
On success, `RunPrepared` returns the existing caller-owned `RunResult`. Its
|
||||||
|
source-derived provenance must match `Details`, including prompt identity and
|
||||||
|
hash, rendered-prompt hash, session ID, selected profile and backend,
|
||||||
|
effective target, and input hashes.
|
||||||
|
|
||||||
|
`RunResult.StartTime`, `EndTime`, and `Duration` describe the
|
||||||
|
`RunPrepared` execution attempt. They exclude preparation time and any delay
|
||||||
|
while the consumer retained the handle. Preparation timing remains in
|
||||||
|
`PreparedRun`.
|
||||||
|
|
||||||
|
Preparation failure returns no handle. After successful preparation,
|
||||||
|
`RunPrepared` retains the existing rule that an operational failure returns no
|
||||||
|
partial `RunResult`; the consumer already has independent preparation details.
|
||||||
|
A completed content-validation failure remains a successful result with
|
||||||
|
`ValidationFailed`.
|
||||||
|
|
||||||
|
`PrepareExecution` preserves the public error categories of `Prepare`.
|
||||||
|
`RunPrepared` preserves applicable invalid-request, credential, capacity,
|
||||||
|
generation, validation, collaborator, and cancellation identities without
|
||||||
|
reintroducing source-loading or rendering failures from frozen state.
|
||||||
|
|
||||||
|
## Compatibility And Existing Workflows
|
||||||
|
|
||||||
|
This feature is additive:
|
||||||
|
|
||||||
|
- `Prepare` remains the simple preparation-only operation;
|
||||||
|
- `Run` remains the simple prepare-and-execute operation with its current
|
||||||
|
early-admission and error-ordering behavior;
|
||||||
|
- `PreparedRun` and `RunResult` retain their existing stable JSON
|
||||||
|
representations;
|
||||||
|
- model-client and artifact-reader extension interfaces remain unchanged; and
|
||||||
|
- backend routing, concurrency limits, queue capacities, and provider wire
|
||||||
|
behavior remain unchanged.
|
||||||
|
|
||||||
|
The new workflow may share internal machinery with `Prepare` and `Run`, but it
|
||||||
|
must not change their observable behavior merely to simplify implementation.
|
||||||
|
|
||||||
|
## Documentation
|
||||||
|
|
||||||
|
The completed documentation set has these ownership boundaries:
|
||||||
|
|
||||||
|
- exported declarations and GoDoc own the exact handle, method, lifecycle,
|
||||||
|
ownership, concurrency, credential, error, and cancellation contracts;
|
||||||
|
- the promptkit consumer guide explains when to use `Prepare`, `Run`, or the
|
||||||
|
two-phase prepared-execution workflow; and
|
||||||
|
- internal runner, source-validation, capacity, and model-client documentation
|
||||||
|
describe the implemented collaborator boundaries without duplicating public
|
||||||
|
contracts.
|
||||||
|
|
||||||
|
No release document is part of the feature implementation itself. Release
|
||||||
|
guidance is prepared only when the resulting public API is selected for
|
||||||
|
publication.
|
||||||
|
|
||||||
|
## Non-Goals
|
||||||
|
|
||||||
|
This work does not include:
|
||||||
|
|
||||||
|
- serializable, durable, resumable, or cross-process execution handles;
|
||||||
|
- reuse for multiple consumer-initiated executions;
|
||||||
|
- concurrent execution of one handle;
|
||||||
|
- capacity reservation during preparation;
|
||||||
|
- a background task queue, priorities, worker lifecycle, or job status;
|
||||||
|
- retry, backoff, or provider failover policy;
|
||||||
|
- a separate `RunDetailed` convenience method;
|
||||||
|
- adding preparation details or rendered messages to every `RunResult`;
|
||||||
|
- prompt or profile inspection APIs;
|
||||||
|
- structured capacity or generation errors;
|
||||||
|
- freezing environment-variable credential values for the handle lifetime;
|
||||||
|
- snapshotting provider state or mutable behavior inside an injected
|
||||||
|
`LLMClient`;
|
||||||
|
- allowing target, validation, session, variable, or input overrides after
|
||||||
|
preparation; or
|
||||||
|
- changing existing `Prepare`, `Run`, file-format, provider-request, or stable
|
||||||
|
JSON contracts.
|
||||||
|
|
||||||
|
## Target End State
|
||||||
|
|
||||||
|
After this work:
|
||||||
|
|
||||||
|
- consumers can perform and persist preflight before starting provider work;
|
||||||
|
- one prepared handle executes exactly the source-derived prompt, target,
|
||||||
|
schema, inputs, session, and messages described by its public details;
|
||||||
|
- execution never reloads or rerenders consumer sources;
|
||||||
|
- direct credentials remain confined to opaque, explicitly discardable state;
|
||||||
|
- environment credentials are not retained across the preflight boundary;
|
||||||
|
- backend capacity is reserved only when execution begins;
|
||||||
|
- one handle can start at most one execution attempt, including any internal
|
||||||
|
repair calls owned by that attempt;
|
||||||
|
- preparation details remain available after execution success or failure;
|
||||||
|
- existing simple `Prepare` and `Run` consumers remain unaffected; and
|
||||||
|
- Promptkit continues to own reusable execution mechanics without taking on
|
||||||
|
downstream persistence, redaction, retry, or job-management policy.
|
||||||
246
docs/roadmap/profile-inspection.md
Normal file
246
docs/roadmap/profile-inspection.md
Normal file
@@ -0,0 +1,246 @@
|
|||||||
|
# Prompt-Independent Profile Inspection
|
||||||
|
|
||||||
|
**Status:** Accepted.
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Allow consumers to look up one execution profile by ID and inspect its
|
||||||
|
structurally resolved model target without selecting a prompt, supplying
|
||||||
|
placeholder inputs, checking credential availability, or invoking a model.
|
||||||
|
|
||||||
|
This provides a direct configuration-validation boundary for
|
||||||
|
[Notarius](notarius-promptkit-wishlist.md#priority-2-prompt-independent-profile-inspection)
|
||||||
|
and
|
||||||
|
[Weatherreporter](weatherreporter-promptkit-wishlist.md#priority-3-prompt-independent-profile-inspection).
|
||||||
|
|
||||||
|
## Motivation
|
||||||
|
|
||||||
|
Both downstream consumers need to reject invalid configured profile IDs before
|
||||||
|
starting application work. They currently have to construct a synthetic prompt
|
||||||
|
and call `Engine.Prepare` merely to exercise profile loading, backend lookup,
|
||||||
|
and execution-target resolution.
|
||||||
|
|
||||||
|
That workaround couples profile validation to unrelated prompt definitions,
|
||||||
|
fixture inputs, rendering, schema behavior, and current credential
|
||||||
|
availability. Promptkit already owns profile precedence, backend membership,
|
||||||
|
and target resolution, so it should expose that cohesive capability directly.
|
||||||
|
|
||||||
|
## Consumer Workflow
|
||||||
|
|
||||||
|
The target public workflow is:
|
||||||
|
|
||||||
|
```go
|
||||||
|
inspection, err := engine.InspectProfile(ctx, profileID)
|
||||||
|
if err != nil {
|
||||||
|
// Reject or report the configured profile.
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
target := inspection.EffectiveModelParams
|
||||||
|
if target.APIKeyEnv != "" {
|
||||||
|
// Apply application policy for the named environment variable.
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The target public surface is:
|
||||||
|
|
||||||
|
```go
|
||||||
|
type ProfileInspection struct {
|
||||||
|
ProfileID string
|
||||||
|
EffectiveModelParams ExecutionTarget
|
||||||
|
APIKeyRequired bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) InspectProfile(
|
||||||
|
ctx context.Context,
|
||||||
|
profileID string,
|
||||||
|
) (*ProfileInspection, error)
|
||||||
|
```
|
||||||
|
|
||||||
|
The declarations and GoDoc will own the exact implemented contract. The
|
||||||
|
important public shape is exact lookup through the existing engine, one
|
||||||
|
caller-owned inspection value, the resolved `ExecutionTarget`, and an explicit
|
||||||
|
signal for a direct API-key requirement.
|
||||||
|
|
||||||
|
`EffectiveModelParams.BackendID` identifies the selected registered backend
|
||||||
|
and remains empty for endpoint-only profiles.
|
||||||
|
`EffectiveModelParams.APIKeyEnv` reports the effective credential environment
|
||||||
|
variable name. `APIKeyRequired` reports that the profile requires a direct
|
||||||
|
request credential instead. These states are mutually exclusive after normal
|
||||||
|
profile and backend precedence is applied.
|
||||||
|
|
||||||
|
`ProfileInspection` does not need a stable JSON representation. Consumers that
|
||||||
|
persist application configuration or diagnostics can select the fields their
|
||||||
|
own format requires.
|
||||||
|
|
||||||
|
## Lookup And Precedence
|
||||||
|
|
||||||
|
`InspectProfile` requires a non-blank explicit profile ID. It trims surrounding
|
||||||
|
whitespace and otherwise performs the same case-sensitive exact lookup used by
|
||||||
|
ordinary execution.
|
||||||
|
|
||||||
|
Lookup applies the engine's normal profile-source precedence:
|
||||||
|
|
||||||
|
- programmatic profiles supplied through `WithProfiles`;
|
||||||
|
- the configured file, directory, or `fs.FS` profile source; and
|
||||||
|
- the built-in profile catalog.
|
||||||
|
|
||||||
|
A valid higher-precedence match shadows a lower-precedence profile with the
|
||||||
|
same ID. A malformed or unreadable higher-precedence match fails rather than
|
||||||
|
silently falling back. The
|
||||||
|
[profile format reference](../formats.md#profile-definitions) remains the
|
||||||
|
canonical owner of profile-source and file-format behavior.
|
||||||
|
|
||||||
|
Inspection never derives a profile ID from a prompt's `default_profile`; the
|
||||||
|
caller is inspecting one explicitly named profile.
|
||||||
|
|
||||||
|
## Structural Resolution
|
||||||
|
|
||||||
|
Inspection loads and validates the selected profile, verifies that a named
|
||||||
|
backend exists in the engine's immutable backend registry, and applies normal
|
||||||
|
framework-default, backend, and profile precedence to produce the effective
|
||||||
|
target.
|
||||||
|
|
||||||
|
For the same engine state and profile ID, with no per-run execution override,
|
||||||
|
the inspected target must match the target that ordinary preparation would
|
||||||
|
resolve before applying request credentials and checking their availability.
|
||||||
|
This equivalence must use one shared resolution path rather than a second set
|
||||||
|
of precedence rules.
|
||||||
|
|
||||||
|
Structural resolution includes:
|
||||||
|
|
||||||
|
- backend routing identity;
|
||||||
|
- endpoint and model;
|
||||||
|
- sampling, token, timeout, service-tier, and reasoning settings;
|
||||||
|
- the effective credential environment-variable name or direct-key
|
||||||
|
requirement; and
|
||||||
|
- deeply copied provider-specific extra parameters.
|
||||||
|
|
||||||
|
Inspection returns the effective target rather than a raw profile definition.
|
||||||
|
This keeps framework and backend defaults visible to consumers without
|
||||||
|
creating a second public profile-loading interface.
|
||||||
|
|
||||||
|
The result does not include request-override presence because no
|
||||||
|
`ExecutionTargetOverride` participates in inspection.
|
||||||
|
|
||||||
|
## Credentials And Sensitive Data
|
||||||
|
|
||||||
|
Inspection reports credential requirements but never resolves, retains, or
|
||||||
|
returns a credential value.
|
||||||
|
|
||||||
|
The operation does not read the named environment variable and succeeds when
|
||||||
|
that variable is absent or blank. It accepts neither a direct API key nor an
|
||||||
|
API-key environment override. Consumers decide whether credential availability
|
||||||
|
must be enforced during application configuration, while `Prepare`,
|
||||||
|
`PrepareExecution`, `Run`, and `RunPrepared` retain their execution-time
|
||||||
|
credential contracts.
|
||||||
|
|
||||||
|
Error messages, formatting, and returned values must not expose environment
|
||||||
|
values or other resolved secrets. Existing restrictions against raw API keys
|
||||||
|
in profile sources remain unchanged.
|
||||||
|
|
||||||
|
## Ownership, Consistency, And Concurrency
|
||||||
|
|
||||||
|
Each successful call returns a caller-owned snapshot. Mutating the returned
|
||||||
|
target or any nested extra-parameter map or slice cannot affect the engine,
|
||||||
|
later inspection, or later execution.
|
||||||
|
|
||||||
|
Inspection is safe to call concurrently under the engine's existing immutable
|
||||||
|
registry and repository contracts. It does not mutate profile sources or
|
||||||
|
cache a result globally.
|
||||||
|
|
||||||
|
For filesystem-backed sources, an inspection describes the state observed by
|
||||||
|
that call. It does not freeze the profile for a later `Run`; a source may
|
||||||
|
change between operations. Consumers requiring an exact preflight-to-execution
|
||||||
|
snapshot should use the existing prepared-execution workflow.
|
||||||
|
|
||||||
|
## Errors And Cancellation
|
||||||
|
|
||||||
|
The operation uses existing public error categories:
|
||||||
|
|
||||||
|
- a blank profile ID matches `ErrInvalidRequest`;
|
||||||
|
- an absent exact ID matches `ErrProfileNotFound` and not `ErrProfileLoad`;
|
||||||
|
- read, decode, validation, and source-selection failures match
|
||||||
|
`ErrProfileLoad`; and
|
||||||
|
- an unknown referenced backend or an invalid structurally resolved target
|
||||||
|
matches `ErrProfileLoad`.
|
||||||
|
|
||||||
|
Errors should preserve useful underlying collaborator and context identities
|
||||||
|
through `errors.Is` where the existing facade does so, without exposing
|
||||||
|
internal package types. Context cancellation governs inspection and no partial
|
||||||
|
inspection result is returned on failure.
|
||||||
|
|
||||||
|
Missing credential values are not inspection errors. The operation cannot
|
||||||
|
return capacity or model-generation failures because it performs neither
|
||||||
|
backend admission nor generation.
|
||||||
|
|
||||||
|
## Compatibility And Boundaries
|
||||||
|
|
||||||
|
This feature is additive. Existing profile formats, source precedence,
|
||||||
|
backend registration, `Prepare`, prepared execution, and `Run` behavior remain
|
||||||
|
unchanged.
|
||||||
|
|
||||||
|
The method belongs on the root `Engine` facade. Profile repositories and the
|
||||||
|
backend registry remain internal implementation details, and no new public
|
||||||
|
repository interface is introduced.
|
||||||
|
|
||||||
|
Inspection does not require prompt lookup, rendering, artifact loading, schema
|
||||||
|
loading, validation, backend-capacity admission, or model-client access.
|
||||||
|
Engine construction retains its ordinary configuration requirements; this
|
||||||
|
feature does not introduce a separate profile-only engine.
|
||||||
|
|
||||||
|
## Documentation
|
||||||
|
|
||||||
|
The completed documentation set has these ownership boundaries:
|
||||||
|
|
||||||
|
- exported declarations and GoDoc own the exact method, result, ownership,
|
||||||
|
credential, error, and cancellation contracts;
|
||||||
|
- the promptkit consumer guide explains configuration-time profile inspection
|
||||||
|
and distinguishes it from `Prepare` and prepared execution;
|
||||||
|
- the profile format reference continues to own profile fields and source
|
||||||
|
precedence; and
|
||||||
|
- internal documentation describes shared profile and target resolution
|
||||||
|
without duplicating public contracts.
|
||||||
|
|
||||||
|
## Non-Goals
|
||||||
|
|
||||||
|
This work does not include:
|
||||||
|
|
||||||
|
- enumerating or searching profiles;
|
||||||
|
- returning raw profile definitions or profile source paths;
|
||||||
|
- accepting per-run execution overrides, direct API keys, or API-key
|
||||||
|
environment overrides;
|
||||||
|
- checking environment-variable contents or other credential availability;
|
||||||
|
- semantic execution-target fingerprints or profile hashes;
|
||||||
|
- prompt-definition inspection or prompt default-profile resolution;
|
||||||
|
- full-corpus validation across every profile source;
|
||||||
|
- freezing a filesystem-backed profile for later execution;
|
||||||
|
- exposing backend concurrency limits, queue state, or capacity policy;
|
||||||
|
- model generation, provider health checks, or endpoint connectivity tests;
|
||||||
|
- dynamic backend or profile registration after engine construction; or
|
||||||
|
- changing current profile, backend, prepared-execution, or stable JSON
|
||||||
|
contracts.
|
||||||
|
|
||||||
|
## Target End State
|
||||||
|
|
||||||
|
After this work:
|
||||||
|
|
||||||
|
- consumers can validate one configured profile without inventing a prompt or
|
||||||
|
placeholder inputs;
|
||||||
|
- lookup observes ordinary programmatic, configured-source, and built-in
|
||||||
|
precedence;
|
||||||
|
- a successful result proves that the profile exists, is valid, references a
|
||||||
|
registered backend when applicable, and resolves to a structurally valid
|
||||||
|
effective target;
|
||||||
|
- the inspected target matches ordinary preparation for the same profile and
|
||||||
|
engine state before per-run overrides and credential availability checks;
|
||||||
|
- credential requirements are visible without reading or exposing credential
|
||||||
|
values;
|
||||||
|
- returned targets and nested data are caller-owned;
|
||||||
|
- inspection performs no rendering, source loading unrelated to the profile,
|
||||||
|
capacity admission, or model work;
|
||||||
|
- existing execution workflows and compatibility contracts remain unchanged;
|
||||||
|
and
|
||||||
|
- Promptkit owns reusable profile validation while downstream applications
|
||||||
|
retain configuration policy, persistence, logging, and credential-timing
|
||||||
|
decisions.
|
||||||
468
docs/roadmap/weatherreporter-promptkit-wishlist.md
Normal file
468
docs/roadmap/weatherreporter-promptkit-wishlist.md
Normal file
@@ -0,0 +1,468 @@
|
|||||||
|
# Weatherreporter PromptKit Wishlist
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
This document records features and interface changes that would be useful
|
||||||
|
additions to PromptKit from the perspective of the maintainers of
|
||||||
|
Weatherreporter, a downstream application planning to replace its Scriptorium
|
||||||
|
CLI integration with PromptKit.
|
||||||
|
|
||||||
|
PromptKit v0.3.0 provides the capabilities Weatherreporter needs for the
|
||||||
|
migration. None of the ideas below is a hard adoption requirement. They are
|
||||||
|
opportunities to avoid duplicate preparation, validate configuration earlier,
|
||||||
|
improve durable failure diagnostics, and make the integration more direct.
|
||||||
|
|
||||||
|
The examples are API sketches intended to communicate the desired capability,
|
||||||
|
not prescriptive names or finalized Go contracts. The related
|
||||||
|
[Notarius PromptKit wishlist](notarius-promptkit-wishlist.md) proposes several
|
||||||
|
overlapping features from another downstream consumer's perspective.
|
||||||
|
|
||||||
|
## Priority 1: Executable Preparation Handles
|
||||||
|
|
||||||
|
**Disposition:** Accepted into the
|
||||||
|
[executable preparation handles](prepared-execution.md) feature roadmap.
|
||||||
|
|
||||||
|
### Downstream need
|
||||||
|
|
||||||
|
Weatherreporter treats prompt preparation as a durable preflight boundary. It
|
||||||
|
needs to:
|
||||||
|
|
||||||
|
1. prepare the exact request that will be executed;
|
||||||
|
2. persist a safe preparation record before starting the provider call; and
|
||||||
|
3. execute without reloading or rerendering prompt, profile, schema, or input
|
||||||
|
sources.
|
||||||
|
|
||||||
|
Persisting preflight before generation leaves useful evidence when a provider
|
||||||
|
call fails or the process is interrupted during generation.
|
||||||
|
|
||||||
|
### Current integration option
|
||||||
|
|
||||||
|
With PromptKit v0.3.0, Weatherreporter can call `Engine.Prepare`, save selected
|
||||||
|
fields from the returned `PreparedRun`, and then call `Engine.Run` with the
|
||||||
|
same request. Because `Run` performs preparation internally, the work is
|
||||||
|
repeated.
|
||||||
|
|
||||||
|
Weatherreporter plans to use embedded prompt and schema files plus immutable
|
||||||
|
inline input bytes, which removes most of the consistency risk. An external
|
||||||
|
profile file or directory can still change between the two calls, and the
|
||||||
|
second preparation remains unnecessary work.
|
||||||
|
|
||||||
|
The atomic `RunDetailed` operation proposed by the
|
||||||
|
[Notarius wishlist](notarius-promptkit-wishlist.md#priority-1-atomic-execution-with-prepared-details)
|
||||||
|
would guarantee that returned preparation details describe the completed
|
||||||
|
execution. However, returning those details only after generation would not
|
||||||
|
preserve Weatherreporter's preflight-before-generation persistence boundary.
|
||||||
|
|
||||||
|
### Requested capability
|
||||||
|
|
||||||
|
Add an opt-in two-phase API that returns a prepared execution handle:
|
||||||
|
|
||||||
|
```go
|
||||||
|
prepared, err := engine.PrepareExecution(ctx, request)
|
||||||
|
if err != nil {
|
||||||
|
// Handle preparation failure.
|
||||||
|
}
|
||||||
|
|
||||||
|
details := prepared.Details()
|
||||||
|
// Persist a consumer-selected safe preparation record.
|
||||||
|
|
||||||
|
result, err := engine.RunPrepared(ctx, prepared)
|
||||||
|
```
|
||||||
|
|
||||||
|
The exact names and shapes are flexible. The important contract is that
|
||||||
|
`RunPrepared` executes the already prepared prompt and does not reload or
|
||||||
|
rerender its prompt, profile, schema, or input sources.
|
||||||
|
|
||||||
|
`Details` should return the same caller-owned public preparation information
|
||||||
|
currently represented by `PreparedRun`. The execution handle may retain opaque
|
||||||
|
engine-owned state needed to invoke the model and validate the response.
|
||||||
|
|
||||||
|
### Design considerations
|
||||||
|
|
||||||
|
- Keep `Prepare` and `Run` available for consumers that do not need a
|
||||||
|
two-phase execution boundary.
|
||||||
|
- Bind a prepared handle to the engine that constructed it.
|
||||||
|
- Define whether a handle is one-shot, reusable, or safe for concurrent use.
|
||||||
|
A one-shot contract may be the safest initial design.
|
||||||
|
- Do not give the opaque handle a stable JSON representation.
|
||||||
|
- Do not expose or serialize resolved credential values through `Details`.
|
||||||
|
- Define how a direct request API key is retained and released when an opaque
|
||||||
|
handle must carry it until execution.
|
||||||
|
- Preserve caller-owned copies for all public details.
|
||||||
|
- Make context cancellation and backend admission timing explicit.
|
||||||
|
- Document whether profile credential environment values are resolved during
|
||||||
|
preparation or execution.
|
||||||
|
- Ensure an execution error does not invalidate the public details already
|
||||||
|
returned to the consumer.
|
||||||
|
- Consider whether an atomic `RunDetailed` can share the same internal
|
||||||
|
prepared-execution implementation.
|
||||||
|
|
||||||
|
### Value to Weatherreporter
|
||||||
|
|
||||||
|
This is the highest-value upstream addition. It would preserve
|
||||||
|
Weatherreporter's durable preflight behavior, remove duplicate work, eliminate
|
||||||
|
the remaining source-consistency window, and ensure that persisted provenance
|
||||||
|
describes the actual execution.
|
||||||
|
|
||||||
|
## Priority 2: Prompt-Definition Inspection
|
||||||
|
|
||||||
|
**Disposition:** Accepted into the
|
||||||
|
[future catalog](future.md#prompt-definition-inspection).
|
||||||
|
|
||||||
|
### Downstream need
|
||||||
|
|
||||||
|
Weatherreporter has a fixed registry of seven report definitions. Each report
|
||||||
|
selects a prompt ID and one of two output workflows:
|
||||||
|
|
||||||
|
- direct Markdown; or
|
||||||
|
- structured generated text followed by application-owned domain validation
|
||||||
|
and Markdown template rendering.
|
||||||
|
|
||||||
|
Weatherreporter will embed the PromptKit prompt definitions and private
|
||||||
|
response schemas that implement those reports. It needs to validate that the
|
||||||
|
report registry and embedded prompt corpus agree before weather collection or
|
||||||
|
provider execution.
|
||||||
|
|
||||||
|
### Current integration option
|
||||||
|
|
||||||
|
Weatherreporter can maintain synthetic data-package fixtures and call
|
||||||
|
`Engine.Prepare` for every report prompt during tests. Runtime validation can
|
||||||
|
also occur through the ordinary per-report preparation stage.
|
||||||
|
|
||||||
|
This works, but it requires complete placeholder inputs and profile resolution
|
||||||
|
when the application primarily wants to inspect prompt identity and declared
|
||||||
|
contracts.
|
||||||
|
|
||||||
|
### Requested capability
|
||||||
|
|
||||||
|
Add exact prompt-definition lookup without rendering or generation:
|
||||||
|
|
||||||
|
```go
|
||||||
|
type PromptInfo struct {
|
||||||
|
PromptID string
|
||||||
|
PromptVersion string
|
||||||
|
PromptHash string
|
||||||
|
DefaultProfileID string
|
||||||
|
Inputs []InputDefinition
|
||||||
|
OutputContract OutputContract
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) ResolvePrompt(
|
||||||
|
ctx context.Context,
|
||||||
|
promptID string,
|
||||||
|
promptVersion string,
|
||||||
|
) (PromptInfo, error)
|
||||||
|
```
|
||||||
|
|
||||||
|
The exact returned shape may differ. Weatherreporter needs enough information
|
||||||
|
to verify prompt existence, version selection, declared inputs, default
|
||||||
|
profile identity, output format, validation mode, and schema selection without
|
||||||
|
supplying synthetic prompt input.
|
||||||
|
|
||||||
|
### Design considerations
|
||||||
|
|
||||||
|
- Use ordinary PromptKit prompt-source precedence and exact ID/version
|
||||||
|
selection.
|
||||||
|
- Fully load and structurally validate the selected prompt definition.
|
||||||
|
- Validate referenced prompt content files without rendering their templates.
|
||||||
|
- Resolve and validate the selected output contract and schema reference where
|
||||||
|
practical.
|
||||||
|
- Return an opaque prompt-definition equality value rather than raw source
|
||||||
|
bytes.
|
||||||
|
- Do not return rendered messages, schema bodies, profile credentials, or
|
||||||
|
another source of sensitive content.
|
||||||
|
- Preserve typed or sentinel errors for missing and invalid prompts.
|
||||||
|
- Return caller-owned values.
|
||||||
|
- Enumeration of all known prompts is not required for Weatherreporter; exact
|
||||||
|
lookup is sufficient.
|
||||||
|
|
||||||
|
### Value to Weatherreporter
|
||||||
|
|
||||||
|
This would let Weatherreporter directly verify that every report prompt
|
||||||
|
exists, requires the curated `data_package` input, and declares the expected
|
||||||
|
Markdown or JSON Schema output contract. It would reduce synthetic test setup
|
||||||
|
and move failures ahead of weather collection.
|
||||||
|
|
||||||
|
## Priority 3: Prompt-Independent Profile Inspection
|
||||||
|
|
||||||
|
**Disposition:** Covered by the accepted
|
||||||
|
[prompt-independent profile inspection](profile-inspection.md) roadmap.
|
||||||
|
|
||||||
|
### Downstream need
|
||||||
|
|
||||||
|
Weatherreporter will allow operators to select an external PromptKit profile
|
||||||
|
source and may allow an explicit profile override. It should reject a missing
|
||||||
|
profile, unknown backend, malformed execution target, or unsatisfied credential
|
||||||
|
requirement before collecting weather data or writing report artifacts.
|
||||||
|
|
||||||
|
### Current integration option
|
||||||
|
|
||||||
|
Weatherreporter can validate an explicit profile by preparing one embedded
|
||||||
|
prompt with fixture input. Prompts that use their own default profiles can be
|
||||||
|
validated during their normal preparation stage.
|
||||||
|
|
||||||
|
This couples configuration validation to one prompt and requires placeholder
|
||||||
|
input even when only profile and backend resolution are relevant.
|
||||||
|
|
||||||
|
### Requested capability
|
||||||
|
|
||||||
|
The prompt-independent `ResolveProfile` API proposed by the
|
||||||
|
[Notarius wishlist](notarius-promptkit-wishlist.md#priority-2-prompt-independent-profile-inspection)
|
||||||
|
would satisfy this need. It should resolve built-in, file-backed, and
|
||||||
|
programmatic profiles, validate backend membership, report credential
|
||||||
|
requirements without resolving credential values, and preserve typed error
|
||||||
|
classification.
|
||||||
|
|
||||||
|
### Additional Weatherreporter considerations
|
||||||
|
|
||||||
|
- An explicit application profile override should be inspectable without
|
||||||
|
selecting a report prompt.
|
||||||
|
- A prompt-definition inspection result may expose its default profile ID so
|
||||||
|
Weatherreporter can inspect that profile separately.
|
||||||
|
- Inspection should distinguish structural profile validity from current
|
||||||
|
credential availability so configuration validation can apply explicit
|
||||||
|
application policy.
|
||||||
|
- An optional execution-target override should be considered only if it
|
||||||
|
describes the same target that a later run will use.
|
||||||
|
|
||||||
|
### Value to Weatherreporter
|
||||||
|
|
||||||
|
This would improve fail-fast configuration validation and give operator-facing
|
||||||
|
errors direct profile and backend context. It is valuable but not required for
|
||||||
|
the initial migration.
|
||||||
|
|
||||||
|
## Priority 4: Eager Source Validation
|
||||||
|
|
||||||
|
**Disposition:** Deferred until prompt and profile inspection have been used
|
||||||
|
to determine whether a broader engine-wide validation operation is still
|
||||||
|
needed.
|
||||||
|
|
||||||
|
### Downstream need
|
||||||
|
|
||||||
|
PromptKit deliberately defers reading and validating filesystem and `fs.FS`
|
||||||
|
prompt, profile, and schema content until a request needs it. Weatherreporter
|
||||||
|
has a small fixed embedded prompt corpus and one optional external profile
|
||||||
|
source. It would benefit from an explicit offline validation operation for
|
||||||
|
tests, startup diagnostics, and configuration checks.
|
||||||
|
|
||||||
|
### Current integration option
|
||||||
|
|
||||||
|
Weatherreporter can prepare every report prompt with fixture inputs and inspect
|
||||||
|
any explicit profiles individually. That provides strong coverage but requires
|
||||||
|
consumer-maintained traversal and synthetic material.
|
||||||
|
|
||||||
|
### Requested capability
|
||||||
|
|
||||||
|
Consider an opt-in source-validation operation:
|
||||||
|
|
||||||
|
```go
|
||||||
|
type SourceValidationOptions struct {
|
||||||
|
RequireCredentials bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) ValidateSources(
|
||||||
|
ctx context.Context,
|
||||||
|
opts SourceValidationOptions,
|
||||||
|
) error
|
||||||
|
```
|
||||||
|
|
||||||
|
The operation should eagerly discover and structurally validate the configured
|
||||||
|
prompt, profile, and schema sources without model generation.
|
||||||
|
|
||||||
|
### Design considerations
|
||||||
|
|
||||||
|
- Keep deferred validation as the normal `NewEngine` behavior.
|
||||||
|
- Make eager validation an explicit consumer choice.
|
||||||
|
- Validate duplicate IDs and versions, strict YAML decoding, referenced content
|
||||||
|
files, profile/backend membership, schema syntax, and schema references.
|
||||||
|
- Distinguish structural credential declarations from current environment
|
||||||
|
availability.
|
||||||
|
- Do not read or expose credential values when credential availability is not
|
||||||
|
requested.
|
||||||
|
- Preserve source-specific public error identities and useful path context.
|
||||||
|
- Respect context cancellation during filesystem discovery and schema work.
|
||||||
|
- Consider whether exact prompt and profile inspection APIs already provide a
|
||||||
|
smaller sufficient surface before adding an engine-wide operation.
|
||||||
|
|
||||||
|
### Value to Weatherreporter
|
||||||
|
|
||||||
|
This would simplify offline corpus checks and catch malformed operator profile
|
||||||
|
sources before report work begins. It is helpful but lower priority than exact
|
||||||
|
prompt and profile inspection.
|
||||||
|
|
||||||
|
## Priority 5: Structured Generation Errors
|
||||||
|
|
||||||
|
**Disposition:** Deferred pending stronger downstream demand and a narrower
|
||||||
|
design that does not duplicate prepared provenance or impose HTTP-specific
|
||||||
|
fields on injected model clients.
|
||||||
|
|
||||||
|
### Downstream need
|
||||||
|
|
||||||
|
Weatherreporter preserves redacted, inspectable failure receipts for report
|
||||||
|
runs. When model generation fails operationally, it needs to classify the
|
||||||
|
failure and retain safe execution context without parsing error prose.
|
||||||
|
|
||||||
|
Prompt preparation already supplies selected profile, backend, and model
|
||||||
|
identity. Provider status classification would add useful operator context,
|
||||||
|
especially when the built-in OpenAI-compatible client receives a non-success
|
||||||
|
HTTP status.
|
||||||
|
|
||||||
|
### Current integration option
|
||||||
|
|
||||||
|
PromptKit exposes `ErrLLMGenerate` and preserves injected client errors through
|
||||||
|
`errors.Is`. Weatherreporter can reliably classify generation failure and use
|
||||||
|
its preparation record for profile, backend, and model provenance. Any further
|
||||||
|
diagnostic detail remains a redacted error string.
|
||||||
|
|
||||||
|
### Requested capability
|
||||||
|
|
||||||
|
Consider a typed generation error that continues to match `ErrLLMGenerate`:
|
||||||
|
|
||||||
|
```go
|
||||||
|
type GenerationError struct {
|
||||||
|
BackendID string
|
||||||
|
Model string
|
||||||
|
StatusCode int
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The exact fields may differ. The useful contract is safe structured context
|
||||||
|
available through `errors.As`, while `errors.Is(err, ErrLLMGenerate)` remains
|
||||||
|
compatible.
|
||||||
|
|
||||||
|
### Design considerations
|
||||||
|
|
||||||
|
- Include only fields that PromptKit knows reliably and can expose safely.
|
||||||
|
- Treat an HTTP status as optional because injected model clients may not use
|
||||||
|
HTTP.
|
||||||
|
- Do not expose provider response bodies, endpoints, credential environment
|
||||||
|
names, credential values, request content, or generated content.
|
||||||
|
- Do not make a structured error a second source of prompt/profile provenance
|
||||||
|
already present in a prepared execution.
|
||||||
|
- Preserve injected client error identity.
|
||||||
|
- Keep retry and backoff policy with the consuming application.
|
||||||
|
|
||||||
|
### Value to Weatherreporter
|
||||||
|
|
||||||
|
This would improve durable failure receipts and troubleshooting, particularly
|
||||||
|
for built-in transport failures. It is not required if preparation details and
|
||||||
|
the existing sentinel remain available.
|
||||||
|
|
||||||
|
## Lower-Priority Shared Wishlist Items
|
||||||
|
|
||||||
|
### Structured Capacity Errors
|
||||||
|
|
||||||
|
**Disposition:** Accepted into the
|
||||||
|
[future catalog](future.md#structured-capacity-errors).
|
||||||
|
|
||||||
|
The typed capacity error proposed by the
|
||||||
|
[Notarius wishlist](notarius-promptkit-wishlist.md#priority-4-structured-capacity-errors)
|
||||||
|
would improve Weatherreporter diagnostics by exposing the stable backend ID
|
||||||
|
without parsing error text.
|
||||||
|
|
||||||
|
Weatherreporter currently generates batch reports sequentially and constructs
|
||||||
|
one engine per invocation, so engine-local capacity exhaustion is unlikely in
|
||||||
|
the initial design. The feature would become more valuable if report
|
||||||
|
generation later becomes concurrent or PromptKit engines become longer-lived.
|
||||||
|
It should not block adoption.
|
||||||
|
|
||||||
|
### Semantic Execution-Target Fingerprints
|
||||||
|
|
||||||
|
**Disposition:** Deferred until prompt-independent profile inspection defines
|
||||||
|
the resolved target whose configuration identity would be fingerprinted.
|
||||||
|
|
||||||
|
The semantic target digest proposed by the
|
||||||
|
[Notarius wishlist](notarius-promptkit-wishlist.md#priority-3-semantic-execution-target-fingerprints)
|
||||||
|
would provide a compact equality signal for audit metadata.
|
||||||
|
|
||||||
|
Weatherreporter does not currently reuse LLM-dependent checkpoints. Its Recent
|
||||||
|
Changes behavior compares deterministic module snapshots rather than generated
|
||||||
|
reports, so the digest has no immediate cache-correctness role. Existing
|
||||||
|
PromptKit result metadata is sufficient for the initial integration. A digest
|
||||||
|
would still be useful provenance and future-proofing, but it is not a
|
||||||
|
migration priority.
|
||||||
|
|
||||||
|
## Capabilities PromptKit Already Provides Well
|
||||||
|
|
||||||
|
PromptKit v0.3.0 already provides the essential Weatherreporter integration
|
||||||
|
surface:
|
||||||
|
|
||||||
|
- importable in-process engine construction;
|
||||||
|
- filesystem, `fs.FS`, and programmatic prompt, profile, and schema sources;
|
||||||
|
- offline preparation without model execution;
|
||||||
|
- versioned prompt selection;
|
||||||
|
- text, Markdown, JSON, and JSON Schema output contracts;
|
||||||
|
- single-pass output validation with raw output retained after completed
|
||||||
|
validation failure;
|
||||||
|
- inline input artifacts with provenance URIs and input hashes;
|
||||||
|
- selected profile, backend, model, effective target, prompt hashes, timing,
|
||||||
|
and token-usage provenance;
|
||||||
|
- endpoint-only profiles and engine-scoped backend registration;
|
||||||
|
- injected model-client and artifact-reader interfaces;
|
||||||
|
- caller cancellation, generation timeout, and transport timeout behavior; and
|
||||||
|
- public error sentinels for configuration, prompt, profile, artifact,
|
||||||
|
validation, capacity, and generation failures.
|
||||||
|
|
||||||
|
These capabilities are sufficient for Weatherreporter to adopt PromptKit
|
||||||
|
without waiting for new upstream work.
|
||||||
|
|
||||||
|
## Responsibilities That Should Remain In Weatherreporter
|
||||||
|
|
||||||
|
The following concerns belong to Weatherreporter and should not move into
|
||||||
|
PromptKit:
|
||||||
|
|
||||||
|
- report definitions, valid periods, batches, and output naming;
|
||||||
|
- application prompt content and private report response schemas;
|
||||||
|
- deterministic weather facts, modules, and Recent Changes;
|
||||||
|
- curated `data_package` construction and persistence;
|
||||||
|
- generated-text domain validation and Markdown template rendering;
|
||||||
|
- managed artifact paths, atomic writes, metadata, and inspection commands;
|
||||||
|
- preparation, execution, raw-output, and failure-receipt schemas;
|
||||||
|
- CLI configuration loading and precedence;
|
||||||
|
- debug enablement, redaction, placement, sensitivity, and retention;
|
||||||
|
- distributor notification;
|
||||||
|
- batch continuation and any future retry policy; and
|
||||||
|
- application-level compatibility and migration policy.
|
||||||
|
|
||||||
|
## Suggested Upstream Sequence
|
||||||
|
|
||||||
|
If the PromptKit team chooses to pursue these ideas, the most useful order for
|
||||||
|
Weatherreporter would be:
|
||||||
|
|
||||||
|
1. Add executable preparation handles, ideally sharing implementation with an
|
||||||
|
atomic detailed-run API.
|
||||||
|
2. Add prompt-definition inspection.
|
||||||
|
3. Add prompt-independent profile inspection.
|
||||||
|
4. Consider eager source validation after evaluating whether the two exact
|
||||||
|
inspection APIs are sufficient.
|
||||||
|
5. Add structured generation errors.
|
||||||
|
6. Add structured capacity errors and semantic execution-target fingerprints
|
||||||
|
as lower-priority operational improvements.
|
||||||
|
|
||||||
|
The first item removes the only material integration workaround. Prompt and
|
||||||
|
profile inspection improve fail-fast validation. The remaining items improve
|
||||||
|
ergonomics and diagnostics.
|
||||||
|
|
||||||
|
## Adoption Sequencing
|
||||||
|
|
||||||
|
Weatherreporter should not wait for the complete wishlist. PromptKit v0.3.0 is
|
||||||
|
already sufficient when Weatherreporter:
|
||||||
|
|
||||||
|
- embeds immutable prompt and schema assets;
|
||||||
|
- supplies immutable inline data-package bytes;
|
||||||
|
- constructs one engine per CLI invocation;
|
||||||
|
- calls `Prepare` and `Run` with the same request; and
|
||||||
|
- keeps PromptKit behind a weatherreporter-owned adapter contract.
|
||||||
|
|
||||||
|
If executable preparation handles are scheduled for a near-term PromptKit
|
||||||
|
release, Weatherreporter may defer only its final adapter implementation to
|
||||||
|
avoid implementing and then removing duplicate preparation. Prompt corpus
|
||||||
|
retrieval, application-contract design, configuration work, embedded assets,
|
||||||
|
state contracts, and offline fixtures can proceed independently.
|
||||||
|
|
||||||
|
If the feature is not scheduled, Weatherreporter can adopt v0.3.0 and keep the
|
||||||
|
duplicate `Prepare` and `Run` sequence inside its adapter. A later PromptKit
|
||||||
|
upgrade would remain localized behind that neutral boundary.
|
||||||
|
|
||||||
|
Prompt inspection, profile inspection, source validation, structured errors,
|
||||||
|
capacity details, and semantic fingerprints should not gate adoption.
|
||||||
92
engine.go
92
engine.go
@@ -63,9 +63,10 @@ var (
|
|||||||
// ErrPromptRender identifies a failure to render prompt messages or the
|
// ErrPromptRender identifies a failure to render prompt messages or the
|
||||||
// session ID from the resolved inputs and variables.
|
// session ID from the resolved inputs and variables.
|
||||||
ErrPromptRender = errors.New("failed to render prompt")
|
ErrPromptRender = errors.New("failed to render prompt")
|
||||||
// ErrCapacityExceeded identifies a Run rejected because the selected backend
|
// ErrCapacityExceeded identifies a Run or RunPrepared rejected because the
|
||||||
// already admitted ConcurrencyLimit + QueueCapacity calls. It is not an
|
// selected backend already admitted ConcurrencyLimit + QueueCapacity calls.
|
||||||
// invalid request, an LLM or provider rate-limit response, or ErrLLMGenerate.
|
// It is not an invalid request, an LLM or provider rate-limit response, or
|
||||||
|
// ErrLLMGenerate.
|
||||||
ErrCapacityExceeded = errors.New("backend capacity exceeded")
|
ErrCapacityExceeded = errors.New("backend capacity exceeded")
|
||||||
// ErrLLMGenerate identifies a model-client failure or a nil successful
|
// ErrLLMGenerate identifies a model-client failure or a nil successful
|
||||||
// response. Errors returned by an injected LLMClient remain available
|
// response. Errors returned by an injected LLMClient remain available
|
||||||
@@ -79,10 +80,12 @@ var (
|
|||||||
|
|
||||||
// Engine prepares and runs Promptkit prompt requests.
|
// Engine prepares and runs Promptkit prompt requests.
|
||||||
//
|
//
|
||||||
// An Engine is safe for concurrent calls to [Engine.Prepare] and [Engine.Run].
|
// An Engine is safe for concurrent calls to [Engine.Prepare],
|
||||||
// Each Engine owns independent backend-capacity pools that coordinate Run
|
// [Engine.PrepareExecution], [Engine.Run], and [Engine.RunPrepared]. Each
|
||||||
// admission and model generation. Injected collaborators may still be invoked
|
// Engine owns independent backend-capacity pools that coordinate Run and
|
||||||
// concurrently across different backend pools or for unlimited backends.
|
// RunPrepared admission and model generation. Injected collaborators may still
|
||||||
|
// be invoked concurrently across different backend pools or for unlimited
|
||||||
|
// backends.
|
||||||
type Engine struct {
|
type Engine struct {
|
||||||
runner *usecase.Runner
|
runner *usecase.Runner
|
||||||
}
|
}
|
||||||
@@ -146,12 +149,14 @@ type engineOptions struct {
|
|||||||
artifactSource bool
|
artifactSource bool
|
||||||
}
|
}
|
||||||
|
|
||||||
// WithLLMClient replaces the built-in model client used by [Engine.Run].
|
// WithLLMClient replaces the built-in model client used by [Engine.Run] and
|
||||||
|
// [Engine.RunPrepared].
|
||||||
//
|
//
|
||||||
// A nil client makes NewEngine fail with ErrInvalidConfig. The Engine schedules
|
// A nil client makes NewEngine fail with ErrInvalidConfig. The Engine schedules
|
||||||
// Generate calls according to the selected backend's capacity policy, but the
|
// Generate calls according to the selected backend's capacity policy, but the
|
||||||
// client may still be called concurrently across different backend pools or for
|
// client may still be called concurrently across different backend pools or for
|
||||||
// unlimited backends. The client is not used by [Engine.Prepare].
|
// unlimited backends. The client is not used by [Engine.Prepare] or
|
||||||
|
// [Engine.PrepareExecution].
|
||||||
func WithLLMClient(client LLMClient) Option {
|
func WithLLMClient(client LLMClient) Option {
|
||||||
return optionFunc(func(options *engineOptions) error {
|
return optionFunc(func(options *engineOptions) error {
|
||||||
if client == nil {
|
if client == nil {
|
||||||
@@ -457,6 +462,38 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err
|
|||||||
return fromDomainPreparedRun(prepared), nil
|
return fromDomainPreparedRun(prepared), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PrepareExecution completely prepares a prompt request without calling the
|
||||||
|
// configured LLMClient or reserving backend admission capacity.
|
||||||
|
//
|
||||||
|
// The returned opaque handle is bound to this Engine and permits one
|
||||||
|
// [Engine.RunPrepared] invocation. Preparation freezes the selected sources,
|
||||||
|
// rendered messages, effective settings, inputs, provider structured-output
|
||||||
|
// metadata, and validation resources needed by that invocation. The handle
|
||||||
|
// retains a direct RunRequest.APIKey only in private execution state;
|
||||||
|
// [PreparedExecution.Details] is credential-redacted.
|
||||||
|
//
|
||||||
|
// The context governs preparation only. Cancellation after this method
|
||||||
|
// returns does not invalidate the handle or propagate to RunPrepared.
|
||||||
|
// PrepareExecution returns the same error categories as [Engine.Prepare] and
|
||||||
|
// returns no handle on error. A nil Engine returns an error matching
|
||||||
|
// ErrInvalidConfig.
|
||||||
|
func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*PreparedExecution, error) {
|
||||||
|
if e == nil || e.runner == nil {
|
||||||
|
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
|
||||||
|
domainReq, err := toDomainRunRequest(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared, err := e.runner.PrepareExecution(ctx, domainReq)
|
||||||
|
if err != nil {
|
||||||
|
return nil, mapPublicError(err)
|
||||||
|
}
|
||||||
|
return &PreparedExecution{internal: prepared}, nil
|
||||||
|
}
|
||||||
|
|
||||||
// Run prepares a request, invokes the configured LLMClient, and validates the
|
// Run prepares a request, invokes the configured LLMClient, and validates the
|
||||||
// generated output.
|
// generated output.
|
||||||
//
|
//
|
||||||
@@ -491,3 +528,40 @@ func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
|||||||
}
|
}
|
||||||
return fromDomainRunResult(result), nil
|
return fromDomainRunResult(result), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// RunPrepared atomically claims and executes a handle created by
|
||||||
|
// [Engine.PrepareExecution].
|
||||||
|
//
|
||||||
|
// A valid owning-Engine invocation consumes the handle's one attempt before
|
||||||
|
// credential revalidation, backend admission, generation, or validation.
|
||||||
|
// Cancellation, capacity rejection, generation failure, operational
|
||||||
|
// validation failure, and success all leave the handle unusable. A nil,
|
||||||
|
// zero-value, foreign-Engine, discarded, claimed, or used handle returns an
|
||||||
|
// error matching ErrInvalidRequest; a nil Engine returns ErrInvalidConfig and
|
||||||
|
// does not claim the handle.
|
||||||
|
//
|
||||||
|
// The supplied context governs this execution attempt independently of the
|
||||||
|
// preparation context. It covers credential revalidation, admission,
|
||||||
|
// generation, validation, and any internal repair. Result timing begins after
|
||||||
|
// the claim and excludes preparation and consumer-held delay.
|
||||||
|
//
|
||||||
|
// RunPrepared can return ErrInvalidRequest, ErrAPIKeyEnvMissing,
|
||||||
|
// ErrCapacityExceeded, ErrLLMGenerate, or ErrValidation as applicable while
|
||||||
|
// preserving documented collaborator and context identities. A completed
|
||||||
|
// content-validation rejection is returned in RunResult, not as an
|
||||||
|
// operational error. An operational error returns no partial RunResult.
|
||||||
|
func (e *Engine) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*RunResult, error) {
|
||||||
|
if e == nil || e.runner == nil {
|
||||||
|
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||||
|
}
|
||||||
|
|
||||||
|
var internal *usecase.PreparedExecution
|
||||||
|
if prepared != nil {
|
||||||
|
internal = prepared.internal
|
||||||
|
}
|
||||||
|
result, err := e.runner.RunPrepared(ctx, internal)
|
||||||
|
if err != nil {
|
||||||
|
return nil, mapPublicError(err)
|
||||||
|
}
|
||||||
|
return fromDomainRunResult(result), nil
|
||||||
|
}
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
// Package jsonvalue validates and defensively copies JSON-compatible value
|
// Package jsonvalue validates and defensively copies JSON-compatible value
|
||||||
// trees used by public configuration and request boundaries.
|
// trees used by configuration, request, and prepared-state boundaries.
|
||||||
package jsonvalue
|
package jsonvalue
|
||||||
|
|
||||||
import (
|
import (
|
||||||
@@ -18,13 +18,19 @@ type visit struct {
|
|||||||
ptr uintptr
|
ptr uintptr
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Copy validates and deeply copies a JSON-compatible value while preserving
|
||||||
|
// compatible concrete map, slice, array, scalar, and number types.
|
||||||
|
func Copy(src any) (any, error) {
|
||||||
|
return copyValue(reflect.ValueOf(src), "value", make(map[visit]struct{}), true)
|
||||||
|
}
|
||||||
|
|
||||||
// CopyMap validates and deeply copies an extra-parameter map while preserving
|
// CopyMap validates and deeply copies an extra-parameter map while preserving
|
||||||
// compatible concrete map, slice, array, scalar, and number types.
|
// compatible concrete map, slice, array, scalar, and number types.
|
||||||
func CopyMap(src map[string]any) (map[string]any, error) {
|
func CopyMap(src map[string]any) (map[string]any, error) {
|
||||||
if src == nil {
|
if src == nil {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
copied, err := copyValue(reflect.ValueOf(src), "extra_params", make(map[visit]struct{}))
|
copied, err := copyValue(reflect.ValueOf(src), "extra_params", make(map[visit]struct{}), false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -35,7 +41,12 @@ func CopyMap(src map[string]any) (map[string]any, error) {
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func copyValue(value reflect.Value, path string, seen map[visit]struct{}) (any, error) {
|
func copyValue(
|
||||||
|
value reflect.Value,
|
||||||
|
path string,
|
||||||
|
seen map[visit]struct{},
|
||||||
|
allowEmptyMapKeys bool,
|
||||||
|
) (any, error) {
|
||||||
if !value.IsValid() {
|
if !value.IsValid() {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
@@ -43,7 +54,7 @@ func copyValue(value reflect.Value, path string, seen map[visit]struct{}) (any,
|
|||||||
if value.IsNil() {
|
if value.IsNil() {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
return copyValue(value.Elem(), path, seen)
|
return copyValue(value.Elem(), path, seen, allowEmptyMapKeys)
|
||||||
}
|
}
|
||||||
if !value.CanInterface() {
|
if !value.CanInterface() {
|
||||||
return nil, fmt.Errorf("%s: value cannot be copied", path)
|
return nil, fmt.Errorf("%s: value cannot be copied", path)
|
||||||
@@ -88,22 +99,27 @@ func copyValue(value reflect.Value, path string, seen map[visit]struct{}) (any,
|
|||||||
}
|
}
|
||||||
seen[current] = struct{}{}
|
seen[current] = struct{}{}
|
||||||
defer delete(seen, current)
|
defer delete(seen, current)
|
||||||
return copyValue(value.Elem(), path, seen)
|
return copyValue(value.Elem(), path, seen, allowEmptyMapKeys)
|
||||||
case reflect.Map:
|
case reflect.Map:
|
||||||
return copyMapValue(value, path, seen)
|
return copyMapValue(value, path, seen, allowEmptyMapKeys)
|
||||||
case reflect.Slice:
|
case reflect.Slice:
|
||||||
if value.IsNil() {
|
if value.IsNil() {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
return copySequenceValue(value, path, seen)
|
return copySequenceValue(value, path, seen, allowEmptyMapKeys)
|
||||||
case reflect.Array:
|
case reflect.Array:
|
||||||
return copySequenceValue(value, path, seen)
|
return copySequenceValue(value, path, seen, allowEmptyMapKeys)
|
||||||
default:
|
default:
|
||||||
return nil, fmt.Errorf("%s: unsupported JSON value type %s", path, value.Type())
|
return nil, fmt.Errorf("%s: unsupported JSON value type %s", path, value.Type())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func copyMapValue(value reflect.Value, path string, seen map[visit]struct{}) (any, error) {
|
func copyMapValue(
|
||||||
|
value reflect.Value,
|
||||||
|
path string,
|
||||||
|
seen map[visit]struct{},
|
||||||
|
allowEmptyMapKeys bool,
|
||||||
|
) (any, error) {
|
||||||
if value.IsNil() {
|
if value.IsNil() {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
@@ -133,10 +149,10 @@ func copyMapValue(value reflect.Value, path string, seen map[visit]struct{}) (an
|
|||||||
elementType := value.Type().Elem()
|
elementType := value.Type().Elem()
|
||||||
for _, key := range keys {
|
for _, key := range keys {
|
||||||
name := key.String()
|
name := key.String()
|
||||||
if name == "" {
|
if name == "" && !allowEmptyMapKeys {
|
||||||
return nil, fmt.Errorf("%s: map key must not be empty", path)
|
return nil, fmt.Errorf("%s: map key must not be empty", path)
|
||||||
}
|
}
|
||||||
copied, err := copyValue(value.MapIndex(key), path+"."+name, seen)
|
copied, err := copyValue(value.MapIndex(key), path+"."+name, seen, allowEmptyMapKeys)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -171,7 +187,12 @@ func copyMapValue(value reflect.Value, path string, seen map[visit]struct{}) (an
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func copySequenceValue(value reflect.Value, path string, seen map[visit]struct{}) (any, error) {
|
func copySequenceValue(
|
||||||
|
value reflect.Value,
|
||||||
|
path string,
|
||||||
|
seen map[visit]struct{},
|
||||||
|
allowEmptyMapKeys bool,
|
||||||
|
) (any, error) {
|
||||||
var current visit
|
var current visit
|
||||||
if value.Kind() == reflect.Slice {
|
if value.Kind() == reflect.Slice {
|
||||||
current = visit{typ: value.Type(), ptr: value.Pointer()}
|
current = visit{typ: value.Type(), ptr: value.Pointer()}
|
||||||
@@ -186,7 +207,12 @@ func copySequenceValue(value reflect.Value, path string, seen map[visit]struct{}
|
|||||||
preserveType := true
|
preserveType := true
|
||||||
elementType := value.Type().Elem()
|
elementType := value.Type().Elem()
|
||||||
for i := 0; i < value.Len(); i++ {
|
for i := 0; i < value.Len(); i++ {
|
||||||
copied, err := copyValue(value.Index(i), fmt.Sprintf("%s[%d]", path, i), seen)
|
copied, err := copyValue(
|
||||||
|
value.Index(i),
|
||||||
|
fmt.Sprintf("%s[%d]", path, i),
|
||||||
|
seen,
|
||||||
|
allowEmptyMapKeys,
|
||||||
|
)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -44,6 +44,21 @@ func TestCopyMapPreservesTypesAndIsolatesMutations(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCopyAllowsEmptyObjectKeysAndIsolatesMutations(t *testing.T) {
|
||||||
|
nested := map[string]any{"": []any{"original"}}
|
||||||
|
|
||||||
|
copiedValue, err := jsonvalue.Copy(nested)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("copy value: %v", err)
|
||||||
|
}
|
||||||
|
nested[""].([]any)[0] = "changed"
|
||||||
|
|
||||||
|
copied := copiedValue.(map[string]any)
|
||||||
|
if got := copied[""].([]any)[0]; got != "original" {
|
||||||
|
t.Fatalf("copied value was not isolated: %v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestCopyMapRejectsInvalidValues(t *testing.T) {
|
func TestCopyMapRejectsInvalidValues(t *testing.T) {
|
||||||
cyclicMap := map[string]any{}
|
cyclicMap := map[string]any{}
|
||||||
cyclicMap["self"] = cyclicMap
|
cyclicMap["self"] = cyclicMap
|
||||||
|
|||||||
298
internal/usecase/prepared_execution.go
Normal file
298
internal/usecase/prepared_execution.go
Normal file
@@ -0,0 +1,298 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/validate"
|
||||||
|
)
|
||||||
|
|
||||||
|
type preparedExecutionState uint8
|
||||||
|
|
||||||
|
const (
|
||||||
|
preparedExecutionReady preparedExecutionState = iota
|
||||||
|
preparedExecutionClaimed
|
||||||
|
preparedExecutionDiscarded
|
||||||
|
)
|
||||||
|
|
||||||
|
// PreparedExecution owns one frozen, single-use runner execution.
|
||||||
|
type PreparedExecution struct {
|
||||||
|
owner *Runner
|
||||||
|
mu sync.Mutex
|
||||||
|
state preparedExecutionState
|
||||||
|
details *domain.PreparedRun
|
||||||
|
payload *preparedExecutionPayload
|
||||||
|
}
|
||||||
|
|
||||||
|
type preparedExecutionPayload struct {
|
||||||
|
prepared *domain.PreparedRun
|
||||||
|
validation validate.PreparedValidation
|
||||||
|
directKey string
|
||||||
|
}
|
||||||
|
|
||||||
|
// PrepareExecution completes preparation without generation or admission and
|
||||||
|
// returns a runner-bound, single-use execution.
|
||||||
|
func (r *Runner) PrepareExecution(ctx context.Context, req domain.RunRequest) (*PreparedExecution, error) {
|
||||||
|
state, err := r.resolvePreparation(ctx, req, time.Now().UTC())
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
validationPlan, err := r.prepareValidation(ctx, state.effectiveContract)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
structuredOutput, err := r.structuredOutputFromValidationPlan(
|
||||||
|
state.definition,
|
||||||
|
state.effectiveContract,
|
||||||
|
validationPlan,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
prepared, err := r.completePreparationWithStructuredOutput(ctx, req, state, structuredOutput)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
executionSnapshot, err := clonePreparedRun(prepared)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: failed to copy prepared execution: %v", ErrInvalidRequest, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
details, err := clonePreparedRun(executionSnapshot)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: failed to copy prepared execution details: %v", ErrInvalidRequest, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &PreparedExecution{
|
||||||
|
owner: r,
|
||||||
|
state: preparedExecutionReady,
|
||||||
|
details: details,
|
||||||
|
payload: &preparedExecutionPayload{
|
||||||
|
prepared: executionSnapshot,
|
||||||
|
validation: validationPlan,
|
||||||
|
directKey: state.effectiveModel.APIKey,
|
||||||
|
},
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Runner) prepareValidation(
|
||||||
|
ctx context.Context,
|
||||||
|
contract domain.OutputContract,
|
||||||
|
) (validate.PreparedValidation, error) {
|
||||||
|
if r.validator == nil {
|
||||||
|
return noOpPreparedValidation{contract: contract}, nil
|
||||||
|
}
|
||||||
|
preparer, ok := r.validator.(validate.ValidationPreparer)
|
||||||
|
if !ok {
|
||||||
|
return nil, fmt.Errorf("%w: validator does not support prepared validation", ErrValidation)
|
||||||
|
}
|
||||||
|
plan, err := preparer.PrepareValidation(ctx, contract)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
|
||||||
|
}
|
||||||
|
if plan == nil {
|
||||||
|
return nil, fmt.Errorf("%w: validator returned nil prepared validation", ErrValidation)
|
||||||
|
}
|
||||||
|
return plan, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Runner) structuredOutputFromValidationPlan(
|
||||||
|
def *domain.PromptDefinition,
|
||||||
|
contract domain.OutputContract,
|
||||||
|
plan validate.PreparedValidation,
|
||||||
|
) (*domain.StructuredOutputSpec, error) {
|
||||||
|
if contract.ValidationMode != domain.ValidationJSONSchema {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
schemaDocument := plan.SchemaDocument()
|
||||||
|
if schemaDocument == nil {
|
||||||
|
if r.validator == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("%w: prepared json_schema validation has no schema document", ErrValidation)
|
||||||
|
}
|
||||||
|
return structuredOutputSpec(def, schemaDocument), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Details returns a fresh credential-redacted copy of the prepared run.
|
||||||
|
func (p *PreparedExecution) Details() *domain.PreparedRun {
|
||||||
|
if p == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
p.mu.Lock()
|
||||||
|
detailsSnapshot := p.details
|
||||||
|
p.mu.Unlock()
|
||||||
|
|
||||||
|
details, err := clonePreparedRun(detailsSnapshot)
|
||||||
|
if err != nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return details
|
||||||
|
}
|
||||||
|
|
||||||
|
// Discard invalidates an unclaimed execution and drops its private payload.
|
||||||
|
func (p *PreparedExecution) Discard() {
|
||||||
|
if p == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
p.mu.Lock()
|
||||||
|
if p.state != preparedExecutionReady {
|
||||||
|
p.mu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
p.state = preparedExecutionDiscarded
|
||||||
|
payload := p.payload
|
||||||
|
p.payload = nil
|
||||||
|
p.mu.Unlock()
|
||||||
|
|
||||||
|
payload.clear()
|
||||||
|
}
|
||||||
|
|
||||||
|
// RunPrepared claims and executes one prepared execution owned by this runner.
|
||||||
|
func (r *Runner) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*domain.RunResult, error) {
|
||||||
|
payload, err := prepared.claim(r)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer payload.clear()
|
||||||
|
|
||||||
|
runID, err := newRunID()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to create run id: %w", err)
|
||||||
|
}
|
||||||
|
start := time.Now().UTC()
|
||||||
|
|
||||||
|
target := payload.prepared.EffectiveModelParams
|
||||||
|
if err := validateAPIKey(target.APIKeyEnv, payload.directKey, target.APIKeyRequired); err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
release, err := r.admitRun(ctx, target.BackendID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
defer release()
|
||||||
|
|
||||||
|
return r.executePreparedRun(ctx, payload.prepared, payload.directKey, runID, start, func(
|
||||||
|
ctx context.Context,
|
||||||
|
artifact *domain.Artifact,
|
||||||
|
attemptsUsed int,
|
||||||
|
) (domain.ValidationResult, error) {
|
||||||
|
result, validationErr := payload.validation.Validate(ctx, artifact)
|
||||||
|
if validationErr != nil {
|
||||||
|
return domain.ValidationResult{}, validationErr
|
||||||
|
}
|
||||||
|
result.RepairAttempts = attemptsUsed
|
||||||
|
return result, nil
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *PreparedExecution) claim(owner *Runner) (*preparedExecutionPayload, error) {
|
||||||
|
if p == nil || owner == nil || p.owner != owner {
|
||||||
|
return nil, fmt.Errorf("%w: prepared execution does not belong to this runner", ErrInvalidRequest)
|
||||||
|
}
|
||||||
|
|
||||||
|
p.mu.Lock()
|
||||||
|
defer p.mu.Unlock()
|
||||||
|
if p.state != preparedExecutionReady || p.payload == nil {
|
||||||
|
return nil, fmt.Errorf("%w: prepared execution is not ready", ErrInvalidRequest)
|
||||||
|
}
|
||||||
|
p.state = preparedExecutionClaimed
|
||||||
|
payload := p.payload
|
||||||
|
p.payload = nil
|
||||||
|
return payload, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *preparedExecutionPayload) clear() {
|
||||||
|
if p == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if p.prepared != nil {
|
||||||
|
p.prepared.EffectiveModelParams.APIKey = ""
|
||||||
|
}
|
||||||
|
p.prepared = nil
|
||||||
|
p.validation = nil
|
||||||
|
p.directKey = ""
|
||||||
|
}
|
||||||
|
|
||||||
|
type noOpPreparedValidation struct {
|
||||||
|
contract domain.OutputContract
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p noOpPreparedValidation) Validate(
|
||||||
|
ctx context.Context,
|
||||||
|
_ *domain.Artifact,
|
||||||
|
) (domain.ValidationResult, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return domain.ValidationResult{}, err
|
||||||
|
}
|
||||||
|
return domain.ValidationResult{
|
||||||
|
Status: domain.ValidationSkipped,
|
||||||
|
Mode: p.contract.ValidationMode,
|
||||||
|
SchemaPath: p.contract.SchemaPath,
|
||||||
|
RepairAttempts: p.contract.RepairAttempts,
|
||||||
|
IsValid: true,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (noOpPreparedValidation) SchemaDocument() any {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func clonePreparedRun(source *domain.PreparedRun) (*domain.PreparedRun, error) {
|
||||||
|
if source == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
copied := *source
|
||||||
|
extraParams, err := jsonvalue.CopyMap(source.EffectiveModelParams.ExtraParams)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
copied.EffectiveModelParams.ExtraParams = extraParams
|
||||||
|
|
||||||
|
if source.InputHashes != nil {
|
||||||
|
copied.InputHashes = make(map[string]string, len(source.InputHashes))
|
||||||
|
for name, hash := range source.InputHashes {
|
||||||
|
copied.InputHashes[name] = hash
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if source.Messages != nil {
|
||||||
|
copied.Messages = make([]domain.RenderedMessage, len(source.Messages))
|
||||||
|
for i, message := range source.Messages {
|
||||||
|
copied.Messages[i] = message
|
||||||
|
if message.CacheControl != nil {
|
||||||
|
cacheControl := *message.CacheControl
|
||||||
|
copied.Messages[i].CacheControl = &cacheControl
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if source.StructuredOutput != nil {
|
||||||
|
structuredOutput := *source.StructuredOutput
|
||||||
|
copied.StructuredOutput = &structuredOutput
|
||||||
|
if source.StructuredOutput.JSONSchema != nil {
|
||||||
|
jsonSchema := *source.StructuredOutput.JSONSchema
|
||||||
|
copied.StructuredOutput.JSONSchema = &jsonSchema
|
||||||
|
schema, copyErr := cloneJSONValue(source.StructuredOutput.JSONSchema.Schema)
|
||||||
|
if copyErr != nil {
|
||||||
|
return nil, copyErr
|
||||||
|
}
|
||||||
|
copied.StructuredOutput.JSONSchema.Schema = schema
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return &copied, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneJSONValue(source any) (any, error) {
|
||||||
|
return jsonvalue.Copy(source)
|
||||||
|
}
|
||||||
510
internal/usecase/prepared_execution_test.go
Normal file
510
internal/usecase/prepared_execution_test.go
Normal file
@@ -0,0 +1,510 @@
|
|||||||
|
package usecase
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"errors"
|
||||||
|
"os"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/validate"
|
||||||
|
)
|
||||||
|
|
||||||
|
type recordingPreparedValidation struct {
|
||||||
|
contract domain.OutputContract
|
||||||
|
schemaDocument any
|
||||||
|
results []domain.ValidationResult
|
||||||
|
errs []error
|
||||||
|
artifacts []string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *recordingPreparedValidation) Validate(
|
||||||
|
_ context.Context,
|
||||||
|
artifact *domain.Artifact,
|
||||||
|
) (domain.ValidationResult, error) {
|
||||||
|
p.artifacts = append(p.artifacts, string(artifact.Body))
|
||||||
|
index := len(p.artifacts) - 1
|
||||||
|
if index < len(p.errs) && p.errs[index] != nil {
|
||||||
|
return domain.ValidationResult{}, p.errs[index]
|
||||||
|
}
|
||||||
|
if len(p.results) == 0 {
|
||||||
|
return domain.ValidationResult{
|
||||||
|
Status: domain.ValidationPassed,
|
||||||
|
Mode: p.contract.ValidationMode,
|
||||||
|
IsValid: true,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
if index >= len(p.results) {
|
||||||
|
index = len(p.results) - 1
|
||||||
|
}
|
||||||
|
return p.results[index], nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *recordingPreparedValidation) SchemaDocument() any {
|
||||||
|
return p.schemaDocument
|
||||||
|
}
|
||||||
|
|
||||||
|
type recordingValidationPreparer struct {
|
||||||
|
plan *recordingPreparedValidation
|
||||||
|
prepareErr error
|
||||||
|
prepareCalls int
|
||||||
|
directValidateCalls int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *recordingValidationPreparer) Validate(
|
||||||
|
context.Context,
|
||||||
|
*domain.Artifact,
|
||||||
|
domain.OutputContract,
|
||||||
|
) (domain.ValidationResult, error) {
|
||||||
|
v.directValidateCalls++
|
||||||
|
return domain.ValidationResult{}, errors.New("live validation must not be used")
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *recordingValidationPreparer) PrepareValidation(
|
||||||
|
_ context.Context,
|
||||||
|
contract domain.OutputContract,
|
||||||
|
) (validate.PreparedValidation, error) {
|
||||||
|
v.prepareCalls++
|
||||||
|
if v.prepareErr != nil {
|
||||||
|
return nil, v.prepareErr
|
||||||
|
}
|
||||||
|
v.plan.contract = contract
|
||||||
|
return v.plan, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerPrepareExecutionCompletesWithoutAdmissionOrGeneration(t *testing.T) {
|
||||||
|
schemaDocument := map[string]any{
|
||||||
|
"type": "object",
|
||||||
|
"properties": map[string]any{
|
||||||
|
"value": map[string]any{"type": "string"},
|
||||||
|
"": map[string]any{"type": "boolean"},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
|
||||||
|
def.Validation.SchemaPath = "schema.json"
|
||||||
|
reader := defaultArtifactReader()
|
||||||
|
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{
|
||||||
|
SessionID: "prepared-session",
|
||||||
|
Messages: []domain.RenderedMessage{{
|
||||||
|
Role: "user",
|
||||||
|
Content: "original message",
|
||||||
|
}},
|
||||||
|
}}
|
||||||
|
llmClient := &fakeLLM{forbid: true}
|
||||||
|
validator := &recordingValidationPreparer{
|
||||||
|
plan: &recordingPreparedValidation{schemaDocument: schemaDocument},
|
||||||
|
}
|
||||||
|
admitter := &fakeRunAdmitter{}
|
||||||
|
profile := defaultExecutionProfile()
|
||||||
|
profile.ExtraParams = map[string]any{
|
||||||
|
"metadata": map[string]any{"source": "original"},
|
||||||
|
}
|
||||||
|
runner := NewRunner(
|
||||||
|
&fakePromptRepo{def: def},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
|
||||||
|
nil,
|
||||||
|
reader,
|
||||||
|
renderer,
|
||||||
|
llmClient,
|
||||||
|
validator,
|
||||||
|
admitter,
|
||||||
|
)
|
||||||
|
|
||||||
|
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
APIKey: "direct-test-key",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution: %v", err)
|
||||||
|
}
|
||||||
|
defer prepared.Discard()
|
||||||
|
|
||||||
|
if validator.prepareCalls != 1 || validator.directValidateCalls != 0 {
|
||||||
|
t.Fatalf(
|
||||||
|
"validation calls=(prepare=%d direct=%d), want (1, 0)",
|
||||||
|
validator.prepareCalls,
|
||||||
|
validator.directValidateCalls,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if reader.calls != 1 || renderer.calls != 1 {
|
||||||
|
t.Fatalf("completion calls=(artifact=%d render=%d), want (1, 1)", reader.calls, renderer.calls)
|
||||||
|
}
|
||||||
|
if len(admitter.backendIDs) != 0 || llmClient.calls != 0 {
|
||||||
|
t.Fatalf("prepare invoked execution collaborators: admission=%v generation=%d", admitter.backendIDs, llmClient.calls)
|
||||||
|
}
|
||||||
|
|
||||||
|
first := prepared.Details()
|
||||||
|
if first == nil {
|
||||||
|
t.Fatal("prepared details are nil")
|
||||||
|
}
|
||||||
|
if first.EffectiveModelParams.APIKey != "" {
|
||||||
|
t.Fatal("prepared details retained the direct API key")
|
||||||
|
}
|
||||||
|
if first.StructuredOutput == nil ||
|
||||||
|
first.StructuredOutput.JSONSchema == nil ||
|
||||||
|
!reflect.DeepEqual(first.StructuredOutput.JSONSchema.Schema, schemaDocument) {
|
||||||
|
t.Fatalf("prepared details have unexpected structured output: %#v", first.StructuredOutput)
|
||||||
|
}
|
||||||
|
|
||||||
|
first.Messages[0].Content = "caller mutation"
|
||||||
|
first.InputHashes["input"] = "caller mutation"
|
||||||
|
first.EffectiveModelParams.ExtraParams["metadata"].(map[string]any)["source"] = "caller mutation"
|
||||||
|
first.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] = "string"
|
||||||
|
renderer.rendered.Messages[0].Content = "source mutation"
|
||||||
|
|
||||||
|
second := prepared.Details()
|
||||||
|
if second.Messages[0].Content != "original message" ||
|
||||||
|
second.InputHashes["input"] == "caller mutation" ||
|
||||||
|
second.EffectiveModelParams.ExtraParams["metadata"].(map[string]any)["source"] != "original" ||
|
||||||
|
second.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] != "object" {
|
||||||
|
t.Fatalf("details did not preserve an independent snapshot: %#v", second)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerRunPreparedRechecksEnvironmentCredentialBeforeAdmission(t *testing.T) {
|
||||||
|
const environmentName = "PROMPTKIT_PREPARED_EXECUTION_TEST_KEY"
|
||||||
|
t.Setenv(environmentName, "available-during-preparation")
|
||||||
|
|
||||||
|
profile := defaultExecutionProfile()
|
||||||
|
profile.APIKeyEnv = environmentName
|
||||||
|
validator := &recordingValidationPreparer{plan: &recordingPreparedValidation{}}
|
||||||
|
admitter := &fakeRunAdmitter{}
|
||||||
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "unexpected"}}
|
||||||
|
runner := NewRunner(
|
||||||
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
|
||||||
|
nil,
|
||||||
|
defaultArtifactReader(),
|
||||||
|
defaultRenderer(),
|
||||||
|
llmClient,
|
||||||
|
validator,
|
||||||
|
admitter,
|
||||||
|
)
|
||||||
|
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.Unsetenv(environmentName); err != nil {
|
||||||
|
t.Fatalf("unset credential environment: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := runner.RunPrepared(context.Background(), prepared)
|
||||||
|
if result != nil {
|
||||||
|
t.Fatalf("credential failure returned partial result: %+v", result)
|
||||||
|
}
|
||||||
|
if !errors.Is(err, ErrInvalidRequest) || !errors.Is(err, ErrAPIKeyEnvMissing) {
|
||||||
|
t.Fatalf("credential error identities are missing: %v", err)
|
||||||
|
}
|
||||||
|
if len(admitter.backendIDs) != 0 || llmClient.calls != 0 {
|
||||||
|
t.Fatalf("credential failure reached admission or generation: admission=%v generation=%d", admitter.backendIDs, llmClient.calls)
|
||||||
|
}
|
||||||
|
if _, err := runner.RunPrepared(context.Background(), prepared); !errors.Is(err, ErrInvalidRequest) {
|
||||||
|
t.Fatalf("credential failure did not consume execution: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerRunPreparedKeepsDirectCredentialOutOfMetadata(t *testing.T) {
|
||||||
|
const directKey = "direct-prepared-test-key"
|
||||||
|
|
||||||
|
profile := defaultExecutionProfile()
|
||||||
|
profile.APIKeyRequired = true
|
||||||
|
client := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||||
|
runner := NewRunner(
|
||||||
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
|
||||||
|
nil,
|
||||||
|
defaultArtifactReader(),
|
||||||
|
defaultRenderer(),
|
||||||
|
client,
|
||||||
|
&recordingValidationPreparer{plan: &recordingPreparedValidation{}},
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
|
||||||
|
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
APIKey: directKey,
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution: %v", err)
|
||||||
|
}
|
||||||
|
if details := prepared.Details(); details.EffectiveModelParams.APIKey != "" {
|
||||||
|
t.Fatal("prepared details retained direct credential")
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := runner.RunPrepared(context.Background(), prepared)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("run prepared: %v", err)
|
||||||
|
}
|
||||||
|
if client.lastReq.Target.APIKey != directKey {
|
||||||
|
t.Fatal("generation did not receive direct credential")
|
||||||
|
}
|
||||||
|
if result.EffectiveModelParams.APIKey != "" {
|
||||||
|
t.Fatal("run result retained direct credential")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *testing.T) {
|
||||||
|
validator := &recordingValidationPreparer{
|
||||||
|
plan: &recordingPreparedValidation{
|
||||||
|
results: []domain.ValidationResult{
|
||||||
|
{
|
||||||
|
Status: domain.ValidationFailed,
|
||||||
|
Mode: domain.ValidationJSON,
|
||||||
|
Errors: []string{"invalid"},
|
||||||
|
IsValid: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Status: domain.ValidationPassed,
|
||||||
|
Mode: domain.ValidationJSON,
|
||||||
|
IsValid: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
repairer := &fakeRepairer{
|
||||||
|
responses: []*domain.GenerateResponse{{Content: `{"repaired":true}`}},
|
||||||
|
}
|
||||||
|
admitter := &fakeRunAdmitter{}
|
||||||
|
reader := defaultArtifactReader()
|
||||||
|
renderer := defaultRenderer()
|
||||||
|
runner := NewRunnerWithRepairer(
|
||||||
|
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
nil,
|
||||||
|
reader,
|
||||||
|
renderer,
|
||||||
|
&fakeLLM{resp: &domain.GenerateResponse{Content: `{"broken":true}`}},
|
||||||
|
validator,
|
||||||
|
repairer,
|
||||||
|
admitter,
|
||||||
|
)
|
||||||
|
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := runner.RunPrepared(context.Background(), prepared)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("run prepared: %v", err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(validator.plan.artifacts, []string{`{"broken":true}`, `{"repaired":true}`}) {
|
||||||
|
t.Fatalf("prepared validation artifacts=%#v", validator.plan.artifacts)
|
||||||
|
}
|
||||||
|
if validator.directValidateCalls != 0 || repairer.calls != 1 {
|
||||||
|
t.Fatalf("validation/repair calls=(direct=%d repair=%d), want (0, 1)", validator.directValidateCalls, repairer.calls)
|
||||||
|
}
|
||||||
|
if result.Validation.Status != domain.ValidationPassed || result.Validation.RepairAttempts != 1 {
|
||||||
|
t.Fatalf("unexpected repaired validation result: %+v", result.Validation)
|
||||||
|
}
|
||||||
|
if admitter.releaseCalls != 1 {
|
||||||
|
t.Fatalf("admission releases=%d, want 1", admitter.releaseCalls)
|
||||||
|
}
|
||||||
|
if reader.calls != 1 || renderer.calls != 1 {
|
||||||
|
t.Fatalf("execution reopened preparation sources: artifact=%d render=%d", reader.calls, renderer.calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerRunPreparedReleasesAdmissionAcrossExecutionErrors(t *testing.T) {
|
||||||
|
generationFailure := errors.New("generation failed")
|
||||||
|
validationFailure := errors.New("validation failed")
|
||||||
|
repairFailure := errors.New("repair failed")
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
generationErr error
|
||||||
|
validation *recordingPreparedValidation
|
||||||
|
repairer *fakeRepairer
|
||||||
|
wantError error
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "generation failure",
|
||||||
|
generationErr: generationFailure,
|
||||||
|
validation: &recordingPreparedValidation{},
|
||||||
|
wantError: ErrLLMGenerate,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "validation failure",
|
||||||
|
validation: &recordingPreparedValidation{
|
||||||
|
errs: []error{validationFailure},
|
||||||
|
},
|
||||||
|
wantError: ErrValidation,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "repair failure",
|
||||||
|
validation: &recordingPreparedValidation{
|
||||||
|
results: []domain.ValidationResult{{
|
||||||
|
Status: domain.ValidationFailed,
|
||||||
|
Mode: domain.ValidationJSON,
|
||||||
|
Errors: []string{"invalid"},
|
||||||
|
IsValid: false,
|
||||||
|
}},
|
||||||
|
},
|
||||||
|
repairer: &fakeRepairer{err: repairFailure},
|
||||||
|
wantError: ErrValidation,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
def := promptDef(domain.FormatJSON, domain.ValidationJSON, 1)
|
||||||
|
if test.repairer == nil {
|
||||||
|
def.Validation.RepairAttempts = 0
|
||||||
|
}
|
||||||
|
validator := &recordingValidationPreparer{plan: test.validation}
|
||||||
|
admitter := &fakeRunAdmitter{}
|
||||||
|
runner := NewRunnerWithRepairer(
|
||||||
|
&fakePromptRepo{def: def},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
nil,
|
||||||
|
defaultArtifactReader(),
|
||||||
|
defaultRenderer(),
|
||||||
|
&fakeLLM{
|
||||||
|
resp: &domain.GenerateResponse{Content: `{"value":true}`},
|
||||||
|
err: test.generationErr,
|
||||||
|
},
|
||||||
|
validator,
|
||||||
|
test.repairer,
|
||||||
|
admitter,
|
||||||
|
)
|
||||||
|
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := runner.RunPrepared(context.Background(), prepared)
|
||||||
|
if result != nil || !errors.Is(err, test.wantError) {
|
||||||
|
t.Fatalf("run prepared=(%+v, %v), want %v", result, err, test.wantError)
|
||||||
|
}
|
||||||
|
if len(admitter.backendIDs) != 1 || admitter.releaseCalls != 1 {
|
||||||
|
t.Fatalf(
|
||||||
|
"admission calls=%#v releases=%d, want one each",
|
||||||
|
admitter.backendIDs,
|
||||||
|
admitter.releaseCalls,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerPreparedExecutionOwnershipUseAndDiscard(t *testing.T) {
|
||||||
|
newRunner := func() *Runner {
|
||||||
|
return NewRunner(
|
||||||
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
nil,
|
||||||
|
defaultArtifactReader(),
|
||||||
|
defaultRenderer(),
|
||||||
|
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
|
||||||
|
&recordingValidationPreparer{plan: &recordingPreparedValidation{}},
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
request := domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
}
|
||||||
|
|
||||||
|
owner := newRunner()
|
||||||
|
prepared, err := owner.PrepareExecution(context.Background(), request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := newRunner().RunPrepared(context.Background(), prepared); !errors.Is(err, ErrInvalidRequest) {
|
||||||
|
t.Fatalf("foreign runner error=%v, want ErrInvalidRequest", err)
|
||||||
|
}
|
||||||
|
if _, err := owner.RunPrepared(context.Background(), prepared); err != nil {
|
||||||
|
t.Fatalf("owner run prepared: %v", err)
|
||||||
|
}
|
||||||
|
if _, err := owner.RunPrepared(context.Background(), prepared); !errors.Is(err, ErrInvalidRequest) {
|
||||||
|
t.Fatalf("second owner run error=%v, want ErrInvalidRequest", err)
|
||||||
|
}
|
||||||
|
if prepared.Details() == nil {
|
||||||
|
t.Fatal("details unavailable after execution")
|
||||||
|
}
|
||||||
|
|
||||||
|
discarded, err := owner.PrepareExecution(context.Background(), request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare discarded execution: %v", err)
|
||||||
|
}
|
||||||
|
discarded.Discard()
|
||||||
|
discarded.Discard()
|
||||||
|
if _, err := owner.RunPrepared(context.Background(), discarded); !errors.Is(err, ErrInvalidRequest) {
|
||||||
|
t.Fatalf("discarded execution error=%v, want ErrInvalidRequest", err)
|
||||||
|
}
|
||||||
|
if discarded.Details() == nil {
|
||||||
|
t.Fatal("details unavailable after discard")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerPreparedExecutionWithoutValidatorSkipsValidation(t *testing.T) {
|
||||||
|
runner := NewRunner(
|
||||||
|
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 0)},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
nil,
|
||||||
|
defaultArtifactReader(),
|
||||||
|
defaultRenderer(),
|
||||||
|
&fakeLLM{resp: &domain.GenerateResponse{Content: `{}`}},
|
||||||
|
nil,
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := runner.RunPrepared(context.Background(), prepared)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("run prepared: %v", err)
|
||||||
|
}
|
||||||
|
if result.Validation.Status != domain.ValidationSkipped || !result.Validation.IsValid {
|
||||||
|
t.Fatalf("unexpected no-validator result: %+v", result.Validation)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerPrepareExecutionRequiresValidationPreparer(t *testing.T) {
|
||||||
|
reader := defaultArtifactReader()
|
||||||
|
runner := NewRunner(
|
||||||
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
nil,
|
||||||
|
reader,
|
||||||
|
defaultRenderer(),
|
||||||
|
&fakeLLM{forbid: true},
|
||||||
|
&fakeValidator{},
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
|
||||||
|
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
})
|
||||||
|
if prepared != nil || !errors.Is(err, ErrValidation) {
|
||||||
|
t.Fatalf("prepare execution=(%+v, %v), want ErrValidation", prepared, err)
|
||||||
|
}
|
||||||
|
if reader.calls != 0 {
|
||||||
|
t.Fatalf("unsupported validator allowed completion, artifact calls=%d", reader.calls)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -131,29 +131,46 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
|||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
if r.admitter != nil {
|
release, err := r.admitRun(ctx, state.effectiveModel.BackendID)
|
||||||
release, admitErr := r.admitter.Admit(ctx, state.effectiveModel.BackendID)
|
if err != nil {
|
||||||
if admitErr != nil {
|
return nil, err
|
||||||
if errors.Is(admitErr, capacity.ErrCapacityExceeded) {
|
|
||||||
return nil, fmt.Errorf(
|
|
||||||
"backend %q admission: %w",
|
|
||||||
state.effectiveModel.BackendID,
|
|
||||||
admitErr,
|
|
||||||
)
|
|
||||||
}
|
|
||||||
return nil, admitErr
|
|
||||||
}
|
|
||||||
defer release()
|
|
||||||
}
|
}
|
||||||
|
defer release()
|
||||||
|
|
||||||
prepared, err := r.completePreparation(ctx, req, state)
|
prepared, err := r.completePreparation(ctx, req, state)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
directAPIKey := state.effectiveModel.APIKey
|
||||||
|
|
||||||
|
return r.executePreparedRun(ctx, prepared, directAPIKey, runID, start, func(
|
||||||
|
ctx context.Context,
|
||||||
|
artifact *domain.Artifact,
|
||||||
|
attemptsUsed int,
|
||||||
|
) (domain.ValidationResult, error) {
|
||||||
|
return r.validateOutput(ctx, artifact, prepared.OutputContract, attemptsUsed)
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
type preparedValidationFunc func(
|
||||||
|
context.Context,
|
||||||
|
*domain.Artifact,
|
||||||
|
int,
|
||||||
|
) (domain.ValidationResult, error)
|
||||||
|
|
||||||
|
func (r *Runner) executePreparedRun(
|
||||||
|
ctx context.Context,
|
||||||
|
prepared *domain.PreparedRun,
|
||||||
|
directAPIKey string,
|
||||||
|
runID string,
|
||||||
|
start time.Time,
|
||||||
|
validateArtifact preparedValidationFunc,
|
||||||
|
) (*domain.RunResult, error) {
|
||||||
|
executionTarget := prepared.EffectiveModelParams
|
||||||
|
executionTarget.APIKey = directAPIKey
|
||||||
genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{
|
genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{
|
||||||
Prompt: domain.RenderedPrompt{SessionID: prepared.SessionID, Messages: prepared.Messages},
|
Prompt: domain.RenderedPrompt{SessionID: prepared.SessionID, Messages: prepared.Messages},
|
||||||
Target: prepared.EffectiveModelParams,
|
Target: executionTarget,
|
||||||
TargetPresence: prepared.TargetPresence,
|
TargetPresence: prepared.TargetPresence,
|
||||||
StructuredOutput: prepared.StructuredOutput,
|
StructuredOutput: prepared.StructuredOutput,
|
||||||
})
|
})
|
||||||
@@ -165,7 +182,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
|||||||
}
|
}
|
||||||
|
|
||||||
outputArtifact := buildOutputArtifact(genResp.Content, prepared.OutputContract.Format)
|
outputArtifact := buildOutputArtifact(genResp.Content, prepared.OutputContract.Format)
|
||||||
validationResult, err := r.validateOutput(ctx, &outputArtifact, prepared.OutputContract, 0)
|
validationResult, err := validateArtifact(ctx, &outputArtifact, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
|
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
|
||||||
}
|
}
|
||||||
@@ -179,7 +196,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
|||||||
PreviousOutput: genResp.Content,
|
PreviousOutput: genResp.Content,
|
||||||
ValidationErrors: validationResult.Errors,
|
ValidationErrors: validationResult.Errors,
|
||||||
SessionID: prepared.SessionID,
|
SessionID: prepared.SessionID,
|
||||||
Target: prepared.EffectiveModelParams,
|
Target: executionTarget,
|
||||||
StructuredOutput: prepared.StructuredOutput,
|
StructuredOutput: prepared.StructuredOutput,
|
||||||
Attempt: attemptsUsed,
|
Attempt: attemptsUsed,
|
||||||
MaxAttempts: prepared.OutputContract.RepairAttempts,
|
MaxAttempts: prepared.OutputContract.RepairAttempts,
|
||||||
@@ -195,7 +212,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
|||||||
genResp = repairResp
|
genResp = repairResp
|
||||||
outputArtifact = buildOutputArtifact(genResp.Content, prepared.OutputContract.Format)
|
outputArtifact = buildOutputArtifact(genResp.Content, prepared.OutputContract.Format)
|
||||||
|
|
||||||
validationResult, err = r.validateOutput(ctx, &outputArtifact, prepared.OutputContract, attemptsUsed)
|
validationResult, err = validateArtifact(ctx, &outputArtifact, attemptsUsed)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
|
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
|
||||||
}
|
}
|
||||||
@@ -203,6 +220,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
|||||||
}
|
}
|
||||||
|
|
||||||
end := time.Now().UTC()
|
end := time.Now().UTC()
|
||||||
|
executionTarget.APIKey = ""
|
||||||
|
|
||||||
return &domain.RunResult{
|
return &domain.RunResult{
|
||||||
RunID: runID,
|
RunID: runID,
|
||||||
@@ -218,7 +236,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
|||||||
SelectedBackendID: prepared.SelectedBackendID,
|
SelectedBackendID: prepared.SelectedBackendID,
|
||||||
ModelName: prepared.EffectiveModelParams.Model,
|
ModelName: prepared.EffectiveModelParams.Model,
|
||||||
Endpoint: prepared.EffectiveModelParams.Endpoint,
|
Endpoint: prepared.EffectiveModelParams.Endpoint,
|
||||||
EffectiveModelParams: prepared.EffectiveModelParams,
|
EffectiveModelParams: executionTarget,
|
||||||
InputHashes: prepared.InputHashes,
|
InputHashes: prepared.InputHashes,
|
||||||
Usage: genResp.Usage,
|
Usage: genResp.Usage,
|
||||||
StartTime: start,
|
StartTime: start,
|
||||||
@@ -324,7 +342,15 @@ func (r *Runner) completePreparation(
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
return r.completePreparationWithStructuredOutput(ctx, req, state, structuredOutput)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *Runner) completePreparationWithStructuredOutput(
|
||||||
|
ctx context.Context,
|
||||||
|
req domain.RunRequest,
|
||||||
|
state *preparationState,
|
||||||
|
structuredOutput *domain.StructuredOutputSpec,
|
||||||
|
) (*domain.PreparedRun, error) {
|
||||||
resolvedInputs := make(map[string]*domain.Artifact, len(req.Inputs))
|
resolvedInputs := make(map[string]*domain.Artifact, len(req.Inputs))
|
||||||
inputHashes := make(map[string]string, len(req.Inputs))
|
inputHashes := make(map[string]string, len(req.Inputs))
|
||||||
for name, ref := range req.Inputs {
|
for name, ref := range req.Inputs {
|
||||||
@@ -354,13 +380,15 @@ func (r *Runner) completePreparation(
|
|||||||
}
|
}
|
||||||
|
|
||||||
end := time.Now().UTC()
|
end := time.Now().UTC()
|
||||||
|
effectiveModel := state.effectiveModel
|
||||||
|
effectiveModel.APIKey = ""
|
||||||
return &domain.PreparedRun{
|
return &domain.PreparedRun{
|
||||||
PromptID: state.definition.ID,
|
PromptID: state.definition.ID,
|
||||||
PromptVersion: state.definition.Version,
|
PromptVersion: state.definition.Version,
|
||||||
PromptHash: state.promptDefinitionHash,
|
PromptHash: state.promptDefinitionHash,
|
||||||
SelectedProfileID: state.selectedProfileID,
|
SelectedProfileID: state.selectedProfileID,
|
||||||
SelectedBackendID: state.effectiveModel.BackendID,
|
SelectedBackendID: state.effectiveModel.BackendID,
|
||||||
EffectiveModelParams: state.effectiveModel,
|
EffectiveModelParams: effectiveModel,
|
||||||
TargetPresence: state.targetPresence,
|
TargetPresence: state.targetPresence,
|
||||||
OutputContract: state.effectiveContract,
|
OutputContract: state.effectiveContract,
|
||||||
StructuredOutput: structuredOutput,
|
StructuredOutput: structuredOutput,
|
||||||
@@ -374,6 +402,20 @@ func (r *Runner) completePreparation(
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (r *Runner) admitRun(ctx context.Context, backendID string) (func(), error) {
|
||||||
|
if r.admitter == nil {
|
||||||
|
return func() {}, nil
|
||||||
|
}
|
||||||
|
release, err := r.admitter.Admit(ctx, backendID)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, capacity.ErrCapacityExceeded) {
|
||||||
|
return nil, fmt.Errorf("backend %q admission: %w", backendID, err)
|
||||||
|
}
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return release, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (r *Runner) resolveStructuredOutput(ctx context.Context, def *domain.PromptDefinition, contract domain.OutputContract) (*domain.StructuredOutputSpec, error) {
|
func (r *Runner) resolveStructuredOutput(ctx context.Context, def *domain.PromptDefinition, contract domain.OutputContract) (*domain.StructuredOutputSpec, error) {
|
||||||
if contract.ValidationMode != domain.ValidationJSONSchema {
|
if contract.ValidationMode != domain.ValidationJSONSchema {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
@@ -389,14 +431,18 @@ func (r *Runner) resolveStructuredOutput(ctx context.Context, def *domain.Prompt
|
|||||||
return nil, fmt.Errorf("%w: failed to load json schema for structured output: %v", ErrValidation, err)
|
return nil, fmt.Errorf("%w: failed to load json schema for structured output: %v", ErrValidation, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
return structuredOutputSpec(def, schemaDoc), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func structuredOutputSpec(def *domain.PromptDefinition, schemaDocument any) *domain.StructuredOutputSpec {
|
||||||
return &domain.StructuredOutputSpec{
|
return &domain.StructuredOutputSpec{
|
||||||
Type: domain.StructuredOutputJSONSchema,
|
Type: domain.StructuredOutputJSONSchema,
|
||||||
JSONSchema: &domain.StructuredOutputJSONSpec{
|
JSONSchema: &domain.StructuredOutputJSONSpec{
|
||||||
Name: deriveStructuredSchemaName(def.ID, def.Version),
|
Name: deriveStructuredSchemaName(def.ID, def.Version),
|
||||||
Strict: true,
|
Strict: true,
|
||||||
Schema: schemaDoc,
|
Schema: schemaDocument,
|
||||||
},
|
},
|
||||||
}, nil
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func deriveStructuredSchemaName(promptID string, promptVersion string) string {
|
func deriveStructuredSchemaName(promptID string, promptVersion string) string {
|
||||||
|
|||||||
@@ -1757,7 +1757,7 @@ func TestRunnerRunDirectAPIKeyBypassesMissingEnvAndReachesLLM(t *testing.T) {
|
|||||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil)
|
||||||
|
|
||||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
result, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
APIKey: directKey,
|
APIKey: directKey,
|
||||||
@@ -1769,6 +1769,9 @@ func TestRunnerRunDirectAPIKeyBypassesMissingEnvAndReachesLLM(t *testing.T) {
|
|||||||
if llmClient.lastReq.Target.APIKey != directKey {
|
if llmClient.lastReq.Target.APIKey != directKey {
|
||||||
t.Fatalf("expected direct API key to reach LLM request")
|
t.Fatalf("expected direct API key to reach LLM request")
|
||||||
}
|
}
|
||||||
|
if result.EffectiveModelParams.APIKey != "" {
|
||||||
|
t.Fatal("run result retained direct API key")
|
||||||
|
}
|
||||||
if llmClient.lastReq.Target.APIKeyEnv != "PROMPTKIT_MISSING_KEY" {
|
if llmClient.lastReq.Target.APIKeyEnv != "PROMPTKIT_MISSING_KEY" {
|
||||||
t.Fatalf("expected api_key_env name to remain on target, got %q", llmClient.lastReq.Target.APIKeyEnv)
|
t.Fatalf("expected api_key_env name to remain on target, got %q", llmClient.lastReq.Target.APIKeyEnv)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -45,6 +45,101 @@ func (v *FSValidator) Validate(ctx context.Context, artifact *domain.Artifact, c
|
|||||||
return validateArtifact(ctx, artifact, contract, v.validateJSONSchema)
|
return validateArtifact(ctx, artifact, contract, v.validateJSONSchema)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type preparedValidation struct {
|
||||||
|
contract domain.OutputContract
|
||||||
|
schemaDocument any
|
||||||
|
schema *jsonschema.Schema
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *preparedValidation) Validate(ctx context.Context, artifact *domain.Artifact) (domain.ValidationResult, error) {
|
||||||
|
return validateArtifact(ctx, artifact, p.contract, p.validateJSONSchema)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *preparedValidation) SchemaDocument() any {
|
||||||
|
return p.schemaDocument
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *preparedValidation) validateJSONSchema(instance any, _ string) ([]string, error) {
|
||||||
|
if p.schema == nil {
|
||||||
|
return nil, errors.New("prepared JSON schema is unavailable")
|
||||||
|
}
|
||||||
|
if err := p.schema.Validate(instance); err != nil {
|
||||||
|
return []string{fmt.Sprintf("json schema validation failed: %v", err)}, nil
|
||||||
|
}
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *StandardValidator) PrepareValidation(ctx context.Context, contract domain.OutputContract) (PreparedValidation, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared := &preparedValidation{contract: contract}
|
||||||
|
if contract.ValidationMode != domain.ValidationJSONSchema {
|
||||||
|
return prepared, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
resolvedSchemaPath, err := v.resolveSchemaPath(contract.SchemaPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
schemaDocument, err := loadJSONSchemaFile(resolvedSchemaPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", resolvedSchemaPath, err)
|
||||||
|
}
|
||||||
|
schemaRoot, err := v.schemaRoot()
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
compiler := newSchemaCompiler(standardSchemaLoader{root: schemaRoot})
|
||||||
|
if err := compiler.AddResource(resolvedSchemaPath, schemaDocument); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to register JSON schema %q: %w", resolvedSchemaPath, err)
|
||||||
|
}
|
||||||
|
schema, err := compiler.Compile(resolvedSchemaPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", resolvedSchemaPath, err)
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared.schemaDocument = schemaDocument
|
||||||
|
prepared.schema = schema
|
||||||
|
return prepared, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v *FSValidator) PrepareValidation(ctx context.Context, contract domain.OutputContract) (PreparedValidation, error) {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared := &preparedValidation{contract: contract}
|
||||||
|
if contract.ValidationMode != domain.ValidationJSONSchema {
|
||||||
|
return prepared, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
schemaName, schemaDocument, err := v.loadSchemaDocument(contract.SchemaPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
resourceURL := fsSchemaResourceURL(schemaName)
|
||||||
|
compiler := newSchemaCompiler(fsSchemaLoader{fsys: v.fsys, root: filecatalog.CleanFSRoot(v.root)})
|
||||||
|
if err := compiler.AddResource(resourceURL, schemaDocument); err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to register JSON schema %q: %w", schemaName, err)
|
||||||
|
}
|
||||||
|
schema, err := compiler.Compile(resourceURL)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", schemaName, err)
|
||||||
|
}
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared.schemaDocument = schemaDocument
|
||||||
|
prepared.schema = schema
|
||||||
|
return prepared, nil
|
||||||
|
}
|
||||||
|
|
||||||
type schemaValidatorFunc func(instance any, schemaPath string) ([]string, error)
|
type schemaValidatorFunc func(instance any, schemaPath string) ([]string, error)
|
||||||
|
|
||||||
func validateArtifact(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract, validateSchema schemaValidatorFunc) (domain.ValidationResult, error) {
|
func validateArtifact(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract, validateSchema schemaValidatorFunc) (domain.ValidationResult, error) {
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"reflect"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
@@ -120,6 +121,68 @@ func TestStandardValidatorJSONSchemaSuccess(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestStandardValidatorPreparedSchemaSurvivesSourceRemoval(t *testing.T) {
|
||||||
|
tmp := t.TempDir()
|
||||||
|
rootSchema := []byte(`{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"title": "original root",
|
||||||
|
"type": "object",
|
||||||
|
"required": ["value"],
|
||||||
|
"properties": {
|
||||||
|
"value": {"$ref": "value.json"}
|
||||||
|
}
|
||||||
|
}`)
|
||||||
|
rootPath := filepath.Join(tmp, "schema.json")
|
||||||
|
referencePath := filepath.Join(tmp, "value.json")
|
||||||
|
if err := os.WriteFile(rootPath, rootSchema, 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.WriteFile(referencePath, []byte(`{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"type": "integer",
|
||||||
|
"minimum": 2
|
||||||
|
}`), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
validator := NewStandardValidator(tmp)
|
||||||
|
preparer, ok := validator.(ValidationPreparer)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("standard validator does not support validation preparation")
|
||||||
|
}
|
||||||
|
prepared, err := preparer.PrepareValidation(context.Background(), domain.OutputContract{
|
||||||
|
ValidationMode: domain.ValidationJSONSchema,
|
||||||
|
SchemaPath: "schema.json",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare validation: %v", err)
|
||||||
|
}
|
||||||
|
assertSchemaDocument(t, prepared.SchemaDocument(), rootSchema)
|
||||||
|
|
||||||
|
if err := os.Remove(rootPath); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := os.Remove(referencePath); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
valid, err := prepared.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"value":3}`)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("validate prepared artifact: %v", err)
|
||||||
|
}
|
||||||
|
if valid.Status != domain.ValidationPassed || !valid.IsValid {
|
||||||
|
t.Fatalf("expected passed/valid, got status=%q valid=%v errors=%v", valid.Status, valid.IsValid, valid.Errors)
|
||||||
|
}
|
||||||
|
|
||||||
|
invalid, err := prepared.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"value":"changed"}`)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("validate prepared artifact: %v", err)
|
||||||
|
}
|
||||||
|
if invalid.Status != domain.ValidationFailed || invalid.IsValid {
|
||||||
|
t.Fatalf("expected failed/invalid, got status=%q valid=%v", invalid.Status, invalid.IsValid)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestStandardValidatorJSONSchemaNestedSchemaPathSuccess(t *testing.T) {
|
func TestStandardValidatorJSONSchemaNestedSchemaPathSuccess(t *testing.T) {
|
||||||
tmp := t.TempDir()
|
tmp := t.TempDir()
|
||||||
nestedDir := filepath.Join(tmp, "dnd")
|
nestedDir := filepath.Join(tmp, "dnd")
|
||||||
@@ -295,6 +358,64 @@ func TestFSValidatorJSONSchemaSuccess(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestFSValidatorPreparedSchemaSurvivesSourceMutation(t *testing.T) {
|
||||||
|
rootSchema := []byte(`{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"title": "original root",
|
||||||
|
"type": "object",
|
||||||
|
"required": ["value"],
|
||||||
|
"properties": {
|
||||||
|
"value": {"$ref": "value.json"}
|
||||||
|
}
|
||||||
|
}`)
|
||||||
|
fsys := fstest.MapFS{
|
||||||
|
"schema.json": &fstest.MapFile{Data: rootSchema},
|
||||||
|
"value.json": &fstest.MapFile{Data: []byte(`{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"type": "integer",
|
||||||
|
"minimum": 2
|
||||||
|
}`)},
|
||||||
|
}
|
||||||
|
validator := NewFSValidator(fsys, ".")
|
||||||
|
preparer, ok := validator.(ValidationPreparer)
|
||||||
|
if !ok {
|
||||||
|
t.Fatal("filesystem validator does not support validation preparation")
|
||||||
|
}
|
||||||
|
prepared, err := preparer.PrepareValidation(context.Background(), domain.OutputContract{
|
||||||
|
ValidationMode: domain.ValidationJSONSchema,
|
||||||
|
SchemaPath: "schema.json",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare validation: %v", err)
|
||||||
|
}
|
||||||
|
assertSchemaDocument(t, prepared.SchemaDocument(), rootSchema)
|
||||||
|
|
||||||
|
fsys["schema.json"] = &fstest.MapFile{Data: []byte(`{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"type": "string"
|
||||||
|
}`)}
|
||||||
|
fsys["value.json"] = &fstest.MapFile{Data: []byte(`{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"type": "string"
|
||||||
|
}`)}
|
||||||
|
|
||||||
|
valid, err := prepared.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"value":3}`)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("validate prepared artifact: %v", err)
|
||||||
|
}
|
||||||
|
if valid.Status != domain.ValidationPassed || !valid.IsValid {
|
||||||
|
t.Fatalf("expected passed/valid, got status=%q valid=%v errors=%v", valid.Status, valid.IsValid, valid.Errors)
|
||||||
|
}
|
||||||
|
|
||||||
|
invalid, err := prepared.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"value":"changed"}`)})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("validate prepared artifact: %v", err)
|
||||||
|
}
|
||||||
|
if invalid.Status != domain.ValidationFailed || invalid.IsValid {
|
||||||
|
t.Fatalf("expected failed/invalid, got status=%q valid=%v", invalid.Status, invalid.IsValid)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestFSValidatorJSONSchemaRegistrationError(t *testing.T) {
|
func TestFSValidatorJSONSchemaRegistrationError(t *testing.T) {
|
||||||
v := NewFSValidator(fstest.MapFS{
|
v := NewFSValidator(fstest.MapFS{
|
||||||
"schemas/%zz.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)},
|
"schemas/%zz.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)},
|
||||||
@@ -548,3 +669,15 @@ func TestJSONSchemaDialectIsDraft2020(t *testing.T) {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func assertSchemaDocument(t *testing.T, got any, expectedJSON []byte) {
|
||||||
|
t.Helper()
|
||||||
|
|
||||||
|
var expected any
|
||||||
|
if err := json.Unmarshal(expectedJSON, &expected); err != nil {
|
||||||
|
t.Fatalf("decode expected schema document: %v", err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got, expected) {
|
||||||
|
t.Fatalf("schema document mismatch:\n got: %#v\nwant: %#v", got, expected)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package validate
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -10,6 +11,20 @@ type Validator interface {
|
|||||||
Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error)
|
Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PreparedValidation validates artifacts against one frozen output contract.
|
||||||
|
type PreparedValidation interface {
|
||||||
|
Validate(ctx context.Context, artifact *domain.Artifact) (domain.ValidationResult, error)
|
||||||
|
// SchemaDocument returns the root JSON Schema document used for provider
|
||||||
|
// structured output, or nil for non-schema modes. Returned internal
|
||||||
|
// immutable state must not be mutated.
|
||||||
|
SchemaDocument() any
|
||||||
|
}
|
||||||
|
|
||||||
|
// ValidationPreparer freezes validation resources for one output contract.
|
||||||
|
type ValidationPreparer interface {
|
||||||
|
PrepareValidation(ctx context.Context, contract domain.OutputContract) (PreparedValidation, error)
|
||||||
|
}
|
||||||
|
|
||||||
// SchemaDocumentLoader loads JSON schema documents using validator path semantics.
|
// SchemaDocumentLoader loads JSON schema documents using validator path semantics.
|
||||||
type SchemaDocumentLoader interface {
|
type SchemaDocumentLoader interface {
|
||||||
LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error)
|
LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error)
|
||||||
|
|||||||
55
prepared_execution.go
Normal file
55
prepared_execution.go
Normal file
@@ -0,0 +1,55 @@
|
|||||||
|
package promptkit
|
||||||
|
|
||||||
|
import "gitea.maximumdirect.net/eric/promptkit/internal/usecase"
|
||||||
|
|
||||||
|
const preparedExecutionString = "promptkit.PreparedExecution{opaque}"
|
||||||
|
|
||||||
|
// PreparedExecution is an opaque, in-process handle for one completely
|
||||||
|
// prepared execution. A handle is bound to the [Engine] that created it and
|
||||||
|
// permits one [Engine.RunPrepared] invocation.
|
||||||
|
//
|
||||||
|
// PreparedExecution contains no supported serializable state and cannot be
|
||||||
|
// used as a restartable job. Copying the value preserves the same shared
|
||||||
|
// lifecycle; it does not create another execution attempt.
|
||||||
|
type PreparedExecution struct {
|
||||||
|
internal *usecase.PreparedExecution
|
||||||
|
}
|
||||||
|
|
||||||
|
// Details returns a fresh caller-owned, credential-redacted copy of the
|
||||||
|
// prepared request details. Mutating the result cannot affect execution or a
|
||||||
|
// later Details call. Details remains available after execution or discard.
|
||||||
|
//
|
||||||
|
// A nil receiver or zero-value PreparedExecution returns a zero [PreparedRun].
|
||||||
|
func (p *PreparedExecution) Details() PreparedRun {
|
||||||
|
if p == nil || p.internal == nil {
|
||||||
|
return PreparedRun{}
|
||||||
|
}
|
||||||
|
details := fromDomainPreparedRun(p.internal.Details())
|
||||||
|
if details == nil {
|
||||||
|
return PreparedRun{}
|
||||||
|
}
|
||||||
|
return *details
|
||||||
|
}
|
||||||
|
|
||||||
|
// Discard invalidates an unclaimed handle and drops Promptkit's references to
|
||||||
|
// its execution-only state. Discard is nil-safe and idempotent. It does not
|
||||||
|
// cancel an execution that has already claimed the handle; use the
|
||||||
|
// [Engine.RunPrepared] context for cancellation.
|
||||||
|
func (p *PreparedExecution) Discard() {
|
||||||
|
if p == nil || p.internal == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
p.internal.Discard()
|
||||||
|
}
|
||||||
|
|
||||||
|
// String returns a constant representation that exposes no retained request,
|
||||||
|
// rendered content, or credential data.
|
||||||
|
func (p *PreparedExecution) String() string {
|
||||||
|
return preparedExecutionString
|
||||||
|
}
|
||||||
|
|
||||||
|
// GoString returns a constant Go-syntax representation that exposes no
|
||||||
|
// retained request, rendered content, or credential data.
|
||||||
|
func (p *PreparedExecution) GoString() string {
|
||||||
|
return preparedExecutionString
|
||||||
|
}
|
||||||
768
prepared_execution_contract_test.go
Normal file
768
prepared_execution_contract_test.go
Normal file
@@ -0,0 +1,768 @@
|
|||||||
|
package promptkit_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
"testing/fstest"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestPreparedExecutionFreezesSourcesAndReturnsIndependentDetails(t *testing.T) {
|
||||||
|
promptSource := preparedPromptSource("original")
|
||||||
|
profileSource := preparedProfileSource("original-model")
|
||||||
|
schemaSource := preparedSchemaSource()
|
||||||
|
reader := &mutablePreparedArtifactReader{
|
||||||
|
body: "original artifact",
|
||||||
|
hash: "original-input-hash",
|
||||||
|
}
|
||||||
|
client := &preparedRecordingClient{
|
||||||
|
response: &promptkit.GenerateResponse{Content: `{"value":3}`},
|
||||||
|
}
|
||||||
|
engine, err := promptkit.NewEngine(
|
||||||
|
promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(promptSource, "."),
|
||||||
|
promptkit.WithProfileFS(profileSource, "."),
|
||||||
|
promptkit.WithSchemaFS(schemaSource, "."),
|
||||||
|
promptkit.WithArtifactReader(reader),
|
||||||
|
promptkit.WithLLMClient(client),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
temperature := 0.25
|
||||||
|
extraParams := map[string]any{
|
||||||
|
"nested": map[string]any{"source": "original"},
|
||||||
|
}
|
||||||
|
request := promptkit.RunRequest{
|
||||||
|
PromptID: "prepared",
|
||||||
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
|
"input": promptkit.Inline("original request input"),
|
||||||
|
},
|
||||||
|
Vars: map[string]string{"label": "original variable"},
|
||||||
|
Execution: &promptkit.ExecutionTargetOverride{
|
||||||
|
Temperature: &temperature,
|
||||||
|
ExtraParams: extraParams,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
preparationContext, cancelPreparation := context.WithCancel(context.Background())
|
||||||
|
prepared, err := engine.PrepareExecution(preparationContext, request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution: %v", err)
|
||||||
|
}
|
||||||
|
cancelPreparation()
|
||||||
|
|
||||||
|
request.PromptID = "changed"
|
||||||
|
request.Inputs["input"] = promptkit.Inline("changed request input")
|
||||||
|
request.Vars["label"] = "changed variable"
|
||||||
|
temperature = 1.5
|
||||||
|
extraParams["nested"].(map[string]any)["source"] = "changed"
|
||||||
|
promptSource["prompt.yaml"] = &fstest.MapFile{Data: []byte(`id: changed`)}
|
||||||
|
profileSource["profile.yaml"] = &fstest.MapFile{Data: []byte(`id: changed`)}
|
||||||
|
schemaSource["schema.json"] = &fstest.MapFile{Data: []byte(`{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"title": "changed root",
|
||||||
|
"type": "string"
|
||||||
|
}`)}
|
||||||
|
schemaSource["value.json"] = &fstest.MapFile{Data: []byte(`{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"type": "string"
|
||||||
|
}`)}
|
||||||
|
reader.set("changed artifact", "changed-input-hash")
|
||||||
|
|
||||||
|
first := prepared.Details()
|
||||||
|
first.Messages[0].Content = "changed details"
|
||||||
|
first.InputHashes["input"] = "changed-details-hash"
|
||||||
|
first.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["source"] = "changed details"
|
||||||
|
first.StructuredOutput.JSONSchema.Schema.(map[string]any)["title"] = "changed details"
|
||||||
|
|
||||||
|
second := prepared.Details()
|
||||||
|
if second.Messages[0].Content != "Input=original artifact Label=original variable" {
|
||||||
|
t.Fatalf("details message changed: %q", second.Messages[0].Content)
|
||||||
|
}
|
||||||
|
if second.InputHashes["input"] != "original-input-hash" {
|
||||||
|
t.Fatalf("details input hash changed: %q", second.InputHashes["input"])
|
||||||
|
}
|
||||||
|
if second.EffectiveModelParams.Model != "original-model" ||
|
||||||
|
second.EffectiveModelParams.Temperature != 0.25 ||
|
||||||
|
second.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["source"] != "original" {
|
||||||
|
t.Fatalf("details target changed: %+v", second.EffectiveModelParams)
|
||||||
|
}
|
||||||
|
schema := second.StructuredOutput.JSONSchema.Schema.(map[string]any)
|
||||||
|
if schema["title"] != "original root" {
|
||||||
|
t.Fatalf("details schema changed: %#v", schema)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := engine.RunPrepared(context.Background(), prepared)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("run prepared after preparation-context cancellation: %v", err)
|
||||||
|
}
|
||||||
|
if result.Validation.Status != promptkit.ValidationPassed || !result.Validation.IsValid {
|
||||||
|
t.Fatalf("frozen schema did not validate original output: %+v", result.Validation)
|
||||||
|
}
|
||||||
|
if reader.callCount() != 1 {
|
||||||
|
t.Fatalf("execution reopened artifact source: calls=%d", reader.callCount())
|
||||||
|
}
|
||||||
|
|
||||||
|
requests := client.snapshot()
|
||||||
|
if len(requests) != 1 {
|
||||||
|
t.Fatalf("generation calls=%d, want 1", len(requests))
|
||||||
|
}
|
||||||
|
generated := requests[0]
|
||||||
|
if generated.Prompt.Messages[0].Content != second.Messages[0].Content ||
|
||||||
|
generated.Target.Model != second.EffectiveModelParams.Model ||
|
||||||
|
!reflect.DeepEqual(generated.Target.ExtraParams, second.EffectiveModelParams.ExtraParams) ||
|
||||||
|
!reflect.DeepEqual(generated.StructuredOutput, second.StructuredOutput) {
|
||||||
|
t.Fatalf("generation did not use frozen details:\nrequest=%+v\ndetails=%+v", generated, second)
|
||||||
|
}
|
||||||
|
if result.PromptID != second.PromptID ||
|
||||||
|
result.PromptVersion != second.PromptVersion ||
|
||||||
|
result.PromptHash != second.PromptHash ||
|
||||||
|
result.SessionID != second.SessionID ||
|
||||||
|
result.RenderedPromptHash != second.RenderedPromptHash ||
|
||||||
|
result.SelectedProfileID != second.SelectedProfileID ||
|
||||||
|
result.SelectedBackendID != second.SelectedBackendID ||
|
||||||
|
!reflect.DeepEqual(result.EffectiveModelParams, second.EffectiveModelParams) ||
|
||||||
|
!reflect.DeepEqual(result.InputHashes, second.InputHashes) {
|
||||||
|
t.Fatalf("result provenance does not match details:\nresult=%+v\ndetails=%+v", result, second)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPreparedExecutionLifecycleAndEngineBinding(t *testing.T) {
|
||||||
|
ownerClient := &preparedRecordingClient{
|
||||||
|
response: &promptkit.GenerateResponse{Content: "ok"},
|
||||||
|
}
|
||||||
|
owner := newPreparedContractEngine(t, ownerClient, "owner content")
|
||||||
|
foreign := newPreparedContractEngine(t, &preparedRecordingClient{
|
||||||
|
response: &promptkit.GenerateResponse{Content: "unexpected"},
|
||||||
|
}, "foreign content")
|
||||||
|
|
||||||
|
prepared, err := owner.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution: %v", err)
|
||||||
|
}
|
||||||
|
copied := *prepared
|
||||||
|
|
||||||
|
var nilEngine *promptkit.Engine
|
||||||
|
if result, err := nilEngine.RunPrepared(context.Background(), prepared); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||||
|
t.Fatalf("nil engine result=(%+v, %v), want ErrInvalidConfig", result, err)
|
||||||
|
}
|
||||||
|
if result, err := foreign.RunPrepared(context.Background(), prepared); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("foreign engine result=(%+v, %v), want ErrInvalidRequest", result, err)
|
||||||
|
}
|
||||||
|
if result, err := owner.RunPrepared(context.Background(), nil); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("nil handle result=(%+v, %v), want ErrInvalidRequest", result, err)
|
||||||
|
}
|
||||||
|
if result, err := owner.RunPrepared(context.Background(), &promptkit.PreparedExecution{}); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("zero handle result=(%+v, %v), want ErrInvalidRequest", result, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := owner.RunPrepared(context.Background(), &copied)
|
||||||
|
if err != nil || result == nil {
|
||||||
|
t.Fatalf("owner run prepared=(%+v, %v), want success", result, err)
|
||||||
|
}
|
||||||
|
for name, handle := range map[string]*promptkit.PreparedExecution{
|
||||||
|
"original": prepared,
|
||||||
|
"copy": &copied,
|
||||||
|
} {
|
||||||
|
if result, err := owner.RunPrepared(context.Background(), handle); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("%s reused handle result=(%+v, %v), want ErrInvalidRequest", name, result, err)
|
||||||
|
}
|
||||||
|
if handle.Details().PromptID != "prepared" {
|
||||||
|
t.Fatalf("%s details unavailable after execution", name)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(ownerClient.snapshot()) != 1 {
|
||||||
|
t.Fatalf("owner generation calls=%d, want 1", len(ownerClient.snapshot()))
|
||||||
|
}
|
||||||
|
|
||||||
|
collaboratorFailure := errors.New("prepared collaborator failure")
|
||||||
|
failingClient := &preparedRecordingClient{err: collaboratorFailure}
|
||||||
|
failingEngine := newPreparedContractEngine(t, failingClient, "failure content")
|
||||||
|
failing, err := failingEngine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare failing execution: %v", err)
|
||||||
|
}
|
||||||
|
if result, err := failingEngine.RunPrepared(context.Background(), failing); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrLLMGenerate) ||
|
||||||
|
!errors.Is(err, collaboratorFailure) {
|
||||||
|
t.Fatalf("generation failure result=(%+v, %v), want public and collaborator identities", result, err)
|
||||||
|
}
|
||||||
|
if result, err := failingEngine.RunPrepared(context.Background(), failing); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("failed execution was reusable: result=(%+v, %v)", result, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cancellationRelease := make(chan struct{})
|
||||||
|
cancellationStarted := make(chan struct{}, 1)
|
||||||
|
cancelingEngine := newPreparedContractEngine(t, &preparedRecordingClient{
|
||||||
|
response: &promptkit.GenerateResponse{Content: "unexpected"},
|
||||||
|
started: cancellationStarted,
|
||||||
|
release: cancellationRelease,
|
||||||
|
}, "cancellation content")
|
||||||
|
canceling, err := cancelingEngine.PrepareExecution(
|
||||||
|
context.Background(),
|
||||||
|
promptkit.RunRequest{PromptID: "prepared"},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare canceled execution: %v", err)
|
||||||
|
}
|
||||||
|
executionContext, cancelExecution := context.WithCancel(context.Background())
|
||||||
|
type canceledOutcome struct {
|
||||||
|
result *promptkit.RunResult
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
canceledResult := make(chan canceledOutcome, 1)
|
||||||
|
go func() {
|
||||||
|
result, runErr := cancelingEngine.RunPrepared(executionContext, canceling)
|
||||||
|
canceledResult <- canceledOutcome{result: result, err: runErr}
|
||||||
|
}()
|
||||||
|
select {
|
||||||
|
case <-cancellationStarted:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("timed out waiting for cancelable generation")
|
||||||
|
}
|
||||||
|
cancelExecution()
|
||||||
|
select {
|
||||||
|
case outcome := <-canceledResult:
|
||||||
|
if outcome.result != nil ||
|
||||||
|
!errors.Is(outcome.err, promptkit.ErrLLMGenerate) ||
|
||||||
|
!errors.Is(outcome.err, context.Canceled) {
|
||||||
|
t.Fatalf(
|
||||||
|
"canceled execution=(%+v, %v), want generation and context identities",
|
||||||
|
outcome.result,
|
||||||
|
outcome.err,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("timed out waiting for canceled execution")
|
||||||
|
}
|
||||||
|
if result, err := cancelingEngine.RunPrepared(context.Background(), canceling); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("canceled execution was reusable: result=(%+v, %v)", result, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPreparedExecutionConcurrentClaimAllowsOneGeneration(t *testing.T) {
|
||||||
|
release := make(chan struct{})
|
||||||
|
client := &preparedRecordingClient{
|
||||||
|
response: &promptkit.GenerateResponse{Content: "ok"},
|
||||||
|
started: make(chan struct{}, 1),
|
||||||
|
release: release,
|
||||||
|
}
|
||||||
|
engine := newPreparedContractEngine(t, client, "concurrent content")
|
||||||
|
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
type outcome struct {
|
||||||
|
result *promptkit.RunResult
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
outcomes := make(chan outcome, 2)
|
||||||
|
for i := 0; i < 2; i++ {
|
||||||
|
go func() {
|
||||||
|
result, runErr := engine.RunPrepared(context.Background(), prepared)
|
||||||
|
outcomes <- outcome{result: result, err: runErr}
|
||||||
|
}()
|
||||||
|
}
|
||||||
|
|
||||||
|
select {
|
||||||
|
case <-client.started:
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("timed out waiting for generation")
|
||||||
|
}
|
||||||
|
select {
|
||||||
|
case loser := <-outcomes:
|
||||||
|
if loser.result != nil || !errors.Is(loser.err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("concurrent loser=(%+v, %v), want ErrInvalidRequest", loser.result, loser.err)
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("timed out waiting for rejected concurrent claim")
|
||||||
|
}
|
||||||
|
|
||||||
|
close(release)
|
||||||
|
select {
|
||||||
|
case winner := <-outcomes:
|
||||||
|
if winner.err != nil || winner.result == nil {
|
||||||
|
t.Fatalf("concurrent winner=(%+v, %v), want success", winner.result, winner.err)
|
||||||
|
}
|
||||||
|
case <-time.After(5 * time.Second):
|
||||||
|
t.Fatal("timed out waiting for successful concurrent claim")
|
||||||
|
}
|
||||||
|
if len(client.snapshot()) != 1 {
|
||||||
|
t.Fatalf("generation calls=%d, want 1", len(client.snapshot()))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPreparedExecutionRunAndDiscardRaceHasOneWinner(t *testing.T) {
|
||||||
|
const attempts = 32
|
||||||
|
|
||||||
|
for i := 0; i < attempts; i++ {
|
||||||
|
client := &preparedRecordingClient{
|
||||||
|
response: &promptkit.GenerateResponse{Content: "ok"},
|
||||||
|
}
|
||||||
|
engine := newPreparedContractEngine(t, client, "race content")
|
||||||
|
prepared, err := engine.PrepareExecution(
|
||||||
|
context.Background(),
|
||||||
|
promptkit.RunRequest{PromptID: "prepared"},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("attempt %d prepare execution: %v", i, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
start := make(chan struct{})
|
||||||
|
type outcome struct {
|
||||||
|
result *promptkit.RunResult
|
||||||
|
err error
|
||||||
|
}
|
||||||
|
runOutcome := make(chan outcome, 1)
|
||||||
|
discardDone := make(chan struct{})
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
<-start
|
||||||
|
result, runErr := engine.RunPrepared(context.Background(), prepared)
|
||||||
|
runOutcome <- outcome{result: result, err: runErr}
|
||||||
|
}()
|
||||||
|
go func() {
|
||||||
|
<-start
|
||||||
|
prepared.Discard()
|
||||||
|
close(discardDone)
|
||||||
|
}()
|
||||||
|
|
||||||
|
close(start)
|
||||||
|
runResult := <-runOutcome
|
||||||
|
<-discardDone
|
||||||
|
calls := len(client.snapshot())
|
||||||
|
switch {
|
||||||
|
case runResult.err == nil:
|
||||||
|
if runResult.result == nil || calls != 1 {
|
||||||
|
t.Fatalf(
|
||||||
|
"attempt %d run won with outcome=(%+v, %v), generation calls=%d",
|
||||||
|
i,
|
||||||
|
runResult.result,
|
||||||
|
runResult.err,
|
||||||
|
calls,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
case errors.Is(runResult.err, promptkit.ErrInvalidRequest):
|
||||||
|
if runResult.result != nil || calls != 0 {
|
||||||
|
t.Fatalf(
|
||||||
|
"attempt %d discard won with outcome=(%+v, %v), generation calls=%d",
|
||||||
|
i,
|
||||||
|
runResult.result,
|
||||||
|
runResult.err,
|
||||||
|
calls,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
t.Fatalf("attempt %d unexpected run outcome=(%+v, %v)", i, runResult.result, runResult.err)
|
||||||
|
}
|
||||||
|
if prepared.Details().PromptID != "prepared" {
|
||||||
|
t.Fatalf("attempt %d details unavailable after race", i)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPreparedExecutionDiscardAndFormattingDoNotExposePrivateState(t *testing.T) {
|
||||||
|
const (
|
||||||
|
directCredential = "pk-test-direct-credential-41f7"
|
||||||
|
renderedContent = "rendered-content-sentinel-98d2"
|
||||||
|
)
|
||||||
|
client := &preparedRecordingClient{
|
||||||
|
response: &promptkit.GenerateResponse{Content: "generated output"},
|
||||||
|
}
|
||||||
|
engine, err := promptkit.NewEngine(
|
||||||
|
promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prepared", "profile", renderedContent), "."),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "profile",
|
||||||
|
Endpoint: "http://example.test/v1",
|
||||||
|
Model: "model",
|
||||||
|
APIKeyRequired: true,
|
||||||
|
}),
|
||||||
|
promptkit.WithLLMClient(client),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine: %v", err)
|
||||||
|
}
|
||||||
|
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
||||||
|
PromptID: "prepared",
|
||||||
|
APIKey: directCredential,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
formattedValues := []string{
|
||||||
|
fmt.Sprint(prepared),
|
||||||
|
fmt.Sprintf("%+v", prepared),
|
||||||
|
fmt.Sprintf("%#v", prepared),
|
||||||
|
}
|
||||||
|
for _, formatted := range formattedValues {
|
||||||
|
if formatted != "promptkit.PreparedExecution{opaque}" {
|
||||||
|
t.Fatalf("unexpected opaque formatting: %q", formatted)
|
||||||
|
}
|
||||||
|
assertPreparedPrivateValuesAbsent(t, formatted, directCredential, renderedContent)
|
||||||
|
}
|
||||||
|
payload, err := json.Marshal(prepared)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal opaque handle: %v", err)
|
||||||
|
}
|
||||||
|
assertPreparedPrivateValuesAbsent(t, string(payload), directCredential, renderedContent)
|
||||||
|
|
||||||
|
detailsBefore := prepared.Details()
|
||||||
|
detailsJSON, err := json.Marshal(detailsBefore)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal prepared details: %v", err)
|
||||||
|
}
|
||||||
|
assertPreparedPrivateValuesAbsent(t, string(detailsJSON), directCredential)
|
||||||
|
|
||||||
|
prepared.Discard()
|
||||||
|
prepared.Discard()
|
||||||
|
result, lifecycleErr := engine.RunPrepared(context.Background(), prepared)
|
||||||
|
if result != nil || !errors.Is(lifecycleErr, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("discarded execution result=(%+v, %v), want ErrInvalidRequest", result, lifecycleErr)
|
||||||
|
}
|
||||||
|
assertPreparedPrivateValuesAbsent(t, lifecycleErr.Error(), directCredential, renderedContent)
|
||||||
|
if !reflect.DeepEqual(prepared.Details(), detailsBefore) {
|
||||||
|
t.Fatal("details changed after discard")
|
||||||
|
}
|
||||||
|
|
||||||
|
executed, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
|
||||||
|
PromptID: "prepared",
|
||||||
|
APIKey: directCredential,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare execution for request inspection: %v", err)
|
||||||
|
}
|
||||||
|
executionResult, err := engine.RunPrepared(context.Background(), executed)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("run execution for request inspection: %v", err)
|
||||||
|
}
|
||||||
|
requests := client.snapshot()
|
||||||
|
if len(requests) != 1 || requests[0].APIKey != directCredential {
|
||||||
|
t.Fatalf("direct credential did not reach only the client credential field: %#v", requests)
|
||||||
|
}
|
||||||
|
requestJSON, err := json.Marshal(requests[0])
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal captured generate request: %v", err)
|
||||||
|
}
|
||||||
|
for _, value := range []string{
|
||||||
|
fmt.Sprint(requests[0]),
|
||||||
|
fmt.Sprintf("%+v", requests[0]),
|
||||||
|
fmt.Sprintf("%#v", requests[0]),
|
||||||
|
string(requestJSON),
|
||||||
|
fmt.Sprint(executionResult),
|
||||||
|
} {
|
||||||
|
assertPreparedPrivateValuesAbsent(t, value, directCredential)
|
||||||
|
}
|
||||||
|
resultJSON, err := json.Marshal(executionResult)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal execution result: %v", err)
|
||||||
|
}
|
||||||
|
assertPreparedPrivateValuesAbsent(t, string(resultJSON), directCredential)
|
||||||
|
|
||||||
|
var nilHandle *promptkit.PreparedExecution
|
||||||
|
nilHandle.Discard()
|
||||||
|
if !reflect.DeepEqual(nilHandle.Details(), promptkit.PreparedRun{}) {
|
||||||
|
t.Fatalf("nil handle details=%+v, want zero value", nilHandle.Details())
|
||||||
|
}
|
||||||
|
zeroHandle := &promptkit.PreparedExecution{}
|
||||||
|
zeroHandle.Discard()
|
||||||
|
if !reflect.DeepEqual(zeroHandle.Details(), promptkit.PreparedRun{}) {
|
||||||
|
t.Fatalf("zero handle details=%+v, want zero value", zeroHandle.Details())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestPreparedExecutionCredentialCapacityAndTimingBoundaries(t *testing.T) {
|
||||||
|
t.Run("credential is rechecked before generation", func(t *testing.T) {
|
||||||
|
const (
|
||||||
|
environmentName = "PROMPTKIT_PREPARED_CONTRACT_KEY"
|
||||||
|
environmentKey = "environment-credential-sentinel"
|
||||||
|
)
|
||||||
|
t.Setenv(environmentName, environmentKey)
|
||||||
|
client := &preparedRecordingClient{
|
||||||
|
response: &promptkit.GenerateResponse{Content: "unexpected"},
|
||||||
|
}
|
||||||
|
engine, err := promptkit.NewEngine(
|
||||||
|
promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prepared", "profile", "content"), "."),
|
||||||
|
promptkit.WithProfileFS(preparedCredentialProfileSource(environmentName), "."),
|
||||||
|
promptkit.WithLLMClient(client),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct credential engine: %v", err)
|
||||||
|
}
|
||||||
|
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare credential execution: %v", err)
|
||||||
|
}
|
||||||
|
if err := os.Unsetenv(environmentName); err != nil {
|
||||||
|
t.Fatalf("unset credential environment: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := engine.RunPrepared(context.Background(), prepared)
|
||||||
|
if result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrInvalidRequest) ||
|
||||||
|
!errors.Is(err, promptkit.ErrAPIKeyEnvMissing) {
|
||||||
|
t.Fatalf("credential execution=(%+v, %v), want credential identities", result, err)
|
||||||
|
}
|
||||||
|
if len(client.snapshot()) != 0 {
|
||||||
|
t.Fatalf("credential failure reached generation: %d calls", len(client.snapshot()))
|
||||||
|
}
|
||||||
|
assertPreparedPrivateValuesAbsent(t, err.Error(), environmentKey)
|
||||||
|
if result, err := engine.RunPrepared(context.Background(), prepared); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("credential failure did not consume handle: result=(%+v, %v)", result, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("preparation does not admit and execution timing starts after retention", func(t *testing.T) {
|
||||||
|
release := make(chan struct{})
|
||||||
|
client := newCapacityGateClient(release, 4)
|
||||||
|
engine := newBackendCapacityEngine(t, client, 1, capacityInt(0), nil)
|
||||||
|
|
||||||
|
activeRun := make(chan capacityRunResult, 1)
|
||||||
|
go runCapacityRequest(
|
||||||
|
engine,
|
||||||
|
context.Background(),
|
||||||
|
promptkit.RunRequest{PromptID: "prompt"},
|
||||||
|
activeRun,
|
||||||
|
)
|
||||||
|
awaitCapacityRequest(t, client.started)
|
||||||
|
|
||||||
|
prepared, err := engine.PrepareExecution(
|
||||||
|
context.Background(),
|
||||||
|
promptkit.RunRequest{PromptID: "prompt"},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare while capacity is full: %v", err)
|
||||||
|
}
|
||||||
|
if _, _, calls := client.snapshot(); calls != 1 {
|
||||||
|
t.Fatalf("preparation invoked generation: calls=%d", calls)
|
||||||
|
}
|
||||||
|
if result, err := engine.RunPrepared(context.Background(), prepared); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrCapacityExceeded) {
|
||||||
|
t.Fatalf("capacity execution=(%+v, %v), want ErrCapacityExceeded", result, err)
|
||||||
|
}
|
||||||
|
if result, err := engine.RunPrepared(context.Background(), prepared); result != nil ||
|
||||||
|
!errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("capacity rejection did not consume handle: result=(%+v, %v)", result, err)
|
||||||
|
}
|
||||||
|
if prepared.Details().PromptID != "prompt" {
|
||||||
|
t.Fatal("details unavailable after capacity rejection")
|
||||||
|
}
|
||||||
|
|
||||||
|
close(release)
|
||||||
|
activeOutcome := awaitCapacityRun(t, activeRun)
|
||||||
|
if activeOutcome.err != nil || activeOutcome.result == nil {
|
||||||
|
t.Fatalf("active run outcome=(%+v, %v), want success", activeOutcome.result, activeOutcome.err)
|
||||||
|
}
|
||||||
|
|
||||||
|
timed, err := engine.PrepareExecution(
|
||||||
|
context.Background(),
|
||||||
|
promptkit.RunRequest{PromptID: "prompt"},
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare timed execution: %v", err)
|
||||||
|
}
|
||||||
|
details := timed.Details()
|
||||||
|
time.Sleep(25 * time.Millisecond)
|
||||||
|
executionFloor := time.Now().UTC()
|
||||||
|
result, err := engine.RunPrepared(context.Background(), timed)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("run timed execution: %v", err)
|
||||||
|
}
|
||||||
|
if result.StartTime.Before(executionFloor) ||
|
||||||
|
!result.StartTime.After(details.EndTime) ||
|
||||||
|
result.EndTime.Before(result.StartTime) ||
|
||||||
|
result.Duration != result.EndTime.Sub(result.StartTime) {
|
||||||
|
t.Fatalf(
|
||||||
|
"execution timing includes preparation or retention: details_end=%s floor=%s result=%+v",
|
||||||
|
details.EndTime,
|
||||||
|
executionFloor,
|
||||||
|
result,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
if _, _, calls := client.snapshot(); calls != 2 {
|
||||||
|
t.Fatalf("generation calls=%d, want active and timed executions only", calls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
type mutablePreparedArtifactReader struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
body string
|
||||||
|
hash string
|
||||||
|
calls int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *mutablePreparedArtifactReader) Read(
|
||||||
|
_ context.Context,
|
||||||
|
_ promptkit.ArtifactRef,
|
||||||
|
) (*promptkit.Artifact, error) {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
r.calls++
|
||||||
|
return &promptkit.Artifact{
|
||||||
|
Body: []byte(r.body),
|
||||||
|
Hash: r.hash,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *mutablePreparedArtifactReader) set(body, hash string) {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
r.body = body
|
||||||
|
r.hash = hash
|
||||||
|
}
|
||||||
|
|
||||||
|
func (r *mutablePreparedArtifactReader) callCount() int {
|
||||||
|
r.mu.Lock()
|
||||||
|
defer r.mu.Unlock()
|
||||||
|
return r.calls
|
||||||
|
}
|
||||||
|
|
||||||
|
type preparedRecordingClient struct {
|
||||||
|
mu sync.Mutex
|
||||||
|
response *promptkit.GenerateResponse
|
||||||
|
err error
|
||||||
|
requests []promptkit.GenerateRequest
|
||||||
|
started chan struct{}
|
||||||
|
release <-chan struct{}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *preparedRecordingClient) Generate(
|
||||||
|
ctx context.Context,
|
||||||
|
request promptkit.GenerateRequest,
|
||||||
|
) (*promptkit.GenerateResponse, error) {
|
||||||
|
c.mu.Lock()
|
||||||
|
c.requests = append(c.requests, request)
|
||||||
|
c.mu.Unlock()
|
||||||
|
|
||||||
|
if c.started != nil {
|
||||||
|
c.started <- struct{}{}
|
||||||
|
}
|
||||||
|
if c.release != nil {
|
||||||
|
select {
|
||||||
|
case <-c.release:
|
||||||
|
case <-ctx.Done():
|
||||||
|
return nil, ctx.Err()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if c.err != nil {
|
||||||
|
return nil, c.err
|
||||||
|
}
|
||||||
|
return c.response, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *preparedRecordingClient) snapshot() []promptkit.GenerateRequest {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
return append([]promptkit.GenerateRequest(nil), c.requests...)
|
||||||
|
}
|
||||||
|
|
||||||
|
func newPreparedContractEngine(
|
||||||
|
t *testing.T,
|
||||||
|
client promptkit.LLMClient,
|
||||||
|
message string,
|
||||||
|
) *promptkit.Engine {
|
||||||
|
t.Helper()
|
||||||
|
engine, err := promptkit.NewEngine(
|
||||||
|
promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prepared", "profile", message), "."),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "profile",
|
||||||
|
Endpoint: "http://example.test/v1",
|
||||||
|
Model: "model",
|
||||||
|
}),
|
||||||
|
promptkit.WithLLMClient(client),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct prepared execution engine: %v", err)
|
||||||
|
}
|
||||||
|
return engine
|
||||||
|
}
|
||||||
|
|
||||||
|
func preparedPromptSource(label string) fstest.MapFS {
|
||||||
|
return fstest.MapFS{
|
||||||
|
"prompt.yaml": &fstest.MapFile{Data: []byte(`id: prepared
|
||||||
|
version: "1"
|
||||||
|
default_profile: profile
|
||||||
|
inputs:
|
||||||
|
- name: input
|
||||||
|
required: true
|
||||||
|
messages:
|
||||||
|
- role: user
|
||||||
|
content: 'Input={{input "input"}} Label={{.label}}'
|
||||||
|
description: ` + label + `
|
||||||
|
output:
|
||||||
|
format: json
|
||||||
|
validation_mode: json_schema
|
||||||
|
schema_path: schema.json
|
||||||
|
`)},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func preparedProfileSource(model string) fstest.MapFS {
|
||||||
|
return fstest.MapFS{
|
||||||
|
"profile.yaml": &fstest.MapFile{Data: []byte(`id: profile
|
||||||
|
endpoint: http://example.test/v1
|
||||||
|
model: ` + model + `
|
||||||
|
`)},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func preparedCredentialProfileSource(environmentName string) fstest.MapFS {
|
||||||
|
return fstest.MapFS{
|
||||||
|
"profile.yaml": &fstest.MapFile{Data: []byte(`id: profile
|
||||||
|
endpoint: http://example.test/v1
|
||||||
|
model: model
|
||||||
|
api_key_env: ` + environmentName + `
|
||||||
|
`)},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func preparedSchemaSource() fstest.MapFS {
|
||||||
|
return fstest.MapFS{
|
||||||
|
"schema.json": &fstest.MapFile{Data: []byte(`{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"title": "original root",
|
||||||
|
"type": "object",
|
||||||
|
"required": ["value"],
|
||||||
|
"properties": {
|
||||||
|
"value": {"$ref": "value.json"}
|
||||||
|
}
|
||||||
|
}`)},
|
||||||
|
"value.json": &fstest.MapFile{Data: []byte(`{
|
||||||
|
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||||
|
"type": "integer",
|
||||||
|
"minimum": 2
|
||||||
|
}`)},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func assertPreparedPrivateValuesAbsent(t *testing.T, value string, privateValues ...string) {
|
||||||
|
t.Helper()
|
||||||
|
for _, privateValue := range privateValues {
|
||||||
|
if strings.Contains(value, privateValue) {
|
||||||
|
t.Fatalf("value exposed private data %q: %s", privateValue, value)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
@@ -138,6 +139,49 @@ func TestUnknownProfileBackendHasProfileLoadIdentity(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLocalBackendConstructsAndRegistersConventionalBackend(t *testing.T) {
|
||||||
|
const (
|
||||||
|
localBackendID = "local"
|
||||||
|
limit = 2
|
||||||
|
)
|
||||||
|
if promptkit.BackendLocal != localBackendID {
|
||||||
|
t.Fatalf("BackendLocal=%q, want %q", promptkit.BackendLocal, localBackendID)
|
||||||
|
}
|
||||||
|
|
||||||
|
endpoint := "http://local.example/v1"
|
||||||
|
backend := promptkit.LocalBackend(endpoint, limit)
|
||||||
|
want := promptkit.Backend{
|
||||||
|
ID: localBackendID,
|
||||||
|
Endpoint: endpoint,
|
||||||
|
ConcurrencyLimit: limit,
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(backend, want) {
|
||||||
|
t.Fatalf("LocalBackend()=%+v, want %+v", backend, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prompt", "local-profile", "message"), "."),
|
||||||
|
promptkit.WithBackend(backend),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "local-profile",
|
||||||
|
BackendID: localBackendID,
|
||||||
|
Model: "model",
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine with local backend: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare with local backend: %v", err)
|
||||||
|
}
|
||||||
|
if prepared.SelectedBackendID != localBackendID ||
|
||||||
|
prepared.EffectiveModelParams.Endpoint != endpoint {
|
||||||
|
t.Fatalf("unexpected local backend preparation: %+v", prepared)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestCustomBackendFlowsThroughProfilesOverridesAndInjectedClient(t *testing.T) {
|
func TestCustomBackendFlowsThroughProfilesOverridesAndInjectedClient(t *testing.T) {
|
||||||
t.Setenv("CUSTOM_LLM_KEY", "test-key")
|
t.Setenv("CUSTOM_LLM_KEY", "test-key")
|
||||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
|
||||||
|
|||||||
63
types.go
63
types.go
@@ -51,7 +51,8 @@ const (
|
|||||||
// ValidationPassed means the generated output satisfied its contract.
|
// ValidationPassed means the generated output satisfied its contract.
|
||||||
ValidationPassed ValidationStatus = "passed"
|
ValidationPassed ValidationStatus = "passed"
|
||||||
// ValidationFailed means validation completed and rejected the generated
|
// ValidationFailed means validation completed and rejected the generated
|
||||||
// output. Engine.Run returns this status in a result, not as an error.
|
// output. Engine.Run and Engine.RunPrepared return this status in a result,
|
||||||
|
// not as an error.
|
||||||
ValidationFailed ValidationStatus = "failed"
|
ValidationFailed ValidationStatus = "failed"
|
||||||
// ValidationSkipped means ValidationNone selected no content check.
|
// ValidationSkipped means ValidationNone selected no content check.
|
||||||
ValidationSkipped ValidationStatus = "skipped"
|
ValidationSkipped ValidationStatus = "skipped"
|
||||||
@@ -78,9 +79,10 @@ const (
|
|||||||
// RunRequest selects one prompt execution. It has no stable JSON
|
// RunRequest selects one prompt execution. It has no stable JSON
|
||||||
// representation.
|
// representation.
|
||||||
//
|
//
|
||||||
// Prepare and Run copy the request's maps, pointers, and nested
|
// Prepare, PrepareExecution, and Run copy the request's maps, pointers, and
|
||||||
// JSON-compatible values before using them. The caller may mutate the request
|
// nested JSON-compatible values before using them. The caller may mutate the
|
||||||
// after either method returns.
|
// request after any method returns. A successful PrepareExecution retains its
|
||||||
|
// own private execution snapshot for RunPrepared.
|
||||||
type RunRequest struct {
|
type RunRequest struct {
|
||||||
// PromptID is the required non-empty prompt identifier.
|
// PromptID is the required non-empty prompt identifier.
|
||||||
PromptID string
|
PromptID string
|
||||||
@@ -98,12 +100,14 @@ type RunRequest struct {
|
|||||||
// opaque consumer metadata, not a credential, and may be exposed in
|
// opaque consumer metadata, not a credential, and may be exposed in
|
||||||
// prepared values, results, collaborator requests, provider requests, and
|
// prepared values, results, collaborator requests, provider requests, and
|
||||||
// provider observability. Callers should use stable, non-sensitive
|
// provider observability. Callers should use stable, non-sensitive
|
||||||
// identifiers. An overlong direct value makes Prepare or Run return an
|
// identifiers. An overlong direct value makes Prepare, PrepareExecution, or
|
||||||
// error matching ErrInvalidRequest.
|
// Run return an error matching ErrInvalidRequest.
|
||||||
SessionID string
|
SessionID string
|
||||||
// APIKey is a request-scoped direct credential. It takes precedence over
|
// APIKey is a request-scoped direct credential. It takes precedence over
|
||||||
// APIKeyEnv, is passed to the selected LLMClient, and is never included in
|
// APIKeyEnv, is passed to the selected LLMClient, and is never included in
|
||||||
// prepared values, results, hashes, JSON, String, or GoString output.
|
// prepared values, results, hashes, JSON, String, or GoString output. A
|
||||||
|
// successful PrepareExecution retains it only in the opaque handle until
|
||||||
|
// RunPrepared claims the handle or Discard invalidates it.
|
||||||
APIKey string `json:"-"`
|
APIKey string `json:"-"`
|
||||||
// Inputs maps prompt input names to references. A nil or empty map is valid
|
// Inputs maps prompt input names to references. A nil or empty map is valid
|
||||||
// only when the selected prompt and its templates require no inputs.
|
// only when the selected prompt and its templates require no inputs.
|
||||||
@@ -119,9 +123,10 @@ type RunRequest struct {
|
|||||||
Validation *OutputContract
|
Validation *OutputContract
|
||||||
}
|
}
|
||||||
|
|
||||||
// PreparedRun contains prepared prompt execution state. It does not include
|
// PreparedRun contains prepared prompt execution state returned by
|
||||||
// resolved API key values, model output, validation results, or internal target
|
// [Engine.Prepare] or [PreparedExecution.Details]. It does not include resolved
|
||||||
// presence metadata. PreparedRun has a stable JSON representation.
|
// API key values, model output, validation results, or internal target presence
|
||||||
|
// metadata. PreparedRun has a stable JSON representation.
|
||||||
//
|
//
|
||||||
// All maps, slices, pointers, and schema values are caller-owned copies. JSON
|
// All maps, slices, pointers, and schema values are caller-owned copies. JSON
|
||||||
// timestamps use RFC 3339 and zero timing values are omitted. Hash formats are
|
// timestamps use RFC 3339 and zero timing values are omitted. Hash formats are
|
||||||
@@ -154,7 +159,8 @@ type PreparedRun struct {
|
|||||||
SessionID string `json:"session_id,omitempty"`
|
SessionID string `json:"session_id,omitempty"`
|
||||||
// RenderedPromptHash is an opaque equality value for SessionID and Messages.
|
// RenderedPromptHash is an opaque equality value for SessionID and Messages.
|
||||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||||
// Messages are the rendered messages that Run passes to the LLM client.
|
// Messages are the rendered messages that Run or RunPrepared passes to the
|
||||||
|
// LLM client.
|
||||||
Messages []RenderedMessage `json:"messages"`
|
Messages []RenderedMessage `json:"messages"`
|
||||||
// StartTime is the UTC time at which preparation began.
|
// StartTime is the UTC time at which preparation began.
|
||||||
StartTime time.Time `json:"start_time,omitempty"`
|
StartTime time.Time `json:"start_time,omitempty"`
|
||||||
@@ -214,12 +220,15 @@ type RunResult struct {
|
|||||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||||
// Usage is the token accounting reported by the LLM client.
|
// Usage is the token accounting reported by the LLM client.
|
||||||
Usage TokenUsage `json:"usage"`
|
Usage TokenUsage `json:"usage"`
|
||||||
// StartTime is the UTC time immediately before preparation begins.
|
// StartTime is the UTC time immediately before ordinary Run preparation or
|
||||||
|
// after RunPrepared claims its handle.
|
||||||
StartTime time.Time `json:"start_time,omitempty"`
|
StartTime time.Time `json:"start_time,omitempty"`
|
||||||
// EndTime is the UTC time after generation and validation complete.
|
// EndTime is the UTC time after generation and validation complete.
|
||||||
EndTime time.Time `json:"end_time,omitempty"`
|
EndTime time.Time `json:"end_time,omitempty"`
|
||||||
// Duration covers preparation, generation, and validation. JSON represents
|
// Duration covers preparation, generation, and validation for Run. For
|
||||||
// it as integer milliseconds in duration_ms and omits a zero value.
|
// RunPrepared it covers only the execution attempt after claim and excludes
|
||||||
|
// preparation and consumer-held delay. JSON represents it as integer
|
||||||
|
// milliseconds in duration_ms and omits a zero value.
|
||||||
Duration time.Duration `json:"-"`
|
Duration time.Duration `json:"-"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -259,10 +268,11 @@ type Artifact struct {
|
|||||||
// ArtifactReader resolves a prompt input reference into its content.
|
// ArtifactReader resolves a prompt input reference into its content.
|
||||||
//
|
//
|
||||||
// Read may be called concurrently. It must honor ctx cancellation to make
|
// Read may be called concurrently. It must honor ctx cancellation to make
|
||||||
// Prepare and Run responsive to cancellation. The engine passes a copied ref
|
// Prepare, PrepareExecution, and Run responsive to cancellation. The engine
|
||||||
// and immediately copies the returned Artifact.Body; it does not retain either
|
// passes a copied ref and immediately copies the returned Artifact.Body; it
|
||||||
// value. Readers supply artifact metadata, and the engine assigns an input-map
|
// does not retain either value. Readers supply artifact metadata, and the
|
||||||
// name only when the returned artifact name is empty.
|
// engine assigns an input-map name only when the returned artifact name is
|
||||||
|
// empty.
|
||||||
//
|
//
|
||||||
// An injected reader owns any application-specific path containment,
|
// An injected reader owns any application-specific path containment,
|
||||||
// authorization, content-size, and content-type policy. It must protect
|
// authorization, content-size, and content-type policy. It must protect
|
||||||
@@ -559,25 +569,26 @@ type StructuredOutputJSONSpec struct {
|
|||||||
Schema any `json:"schema"`
|
Schema any `json:"schema"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// LLMClient executes rendered prompts for [Engine.Run].
|
// LLMClient executes rendered prompts for [Engine.Run] and
|
||||||
|
// [Engine.RunPrepared].
|
||||||
//
|
//
|
||||||
// Generate is scheduled according to the resolved backend's capacity policy.
|
// Generate is scheduled according to the resolved backend's capacity policy.
|
||||||
// It may still be called concurrently for different backend pools or unlimited
|
// It may still be called concurrently for different backend pools or unlimited
|
||||||
// backends. Cancellation while waiting for capacity can prevent Generate from
|
// backends. Cancellation while waiting for capacity can prevent Generate from
|
||||||
// being called. Once invoked, it must honor context cancellation to make Run
|
// being called. Once invoked, it must honor context cancellation to make Run
|
||||||
// responsive to cancellation. The request and all nested maps, slices, and
|
// and RunPrepared responsive to cancellation. The request and all nested maps,
|
||||||
// pointers are client-owned copies and may be mutated or retained without
|
// slices, and pointers are client-owned copies and may be mutated or retained
|
||||||
// affecting engine state.
|
// without affecting engine state.
|
||||||
//
|
//
|
||||||
// Generate receives rendered messages and may receive a direct API key. A
|
// Generate receives rendered messages and may receive a direct API key. A
|
||||||
// client must protect those values and any raw output in its logging, storage,
|
// client must protect those values and any raw output in its logging, storage,
|
||||||
// and retained copies. It is responsible for the cancellation behavior of any
|
// and retained copies. It is responsible for the cancellation behavior of any
|
||||||
// work it starts and for synchronizing access to retained or shared data.
|
// work it starts and for synchronizing access to retained or shared data.
|
||||||
//
|
//
|
||||||
// A returned error makes Run return ErrLLMGenerate while preserving the client
|
// A returned error makes Run or RunPrepared return ErrLLMGenerate while
|
||||||
// error through errors.Is. A nil response with a nil error also produces
|
// preserving the client error through errors.Is. A nil response with a nil
|
||||||
// ErrLLMGenerate. Promptkit copies the non-nil response before returning from
|
// error also produces ErrLLMGenerate. Promptkit copies the non-nil response
|
||||||
// Run.
|
// before returning from either method.
|
||||||
type LLMClient interface {
|
type LLMClient interface {
|
||||||
Generate(context.Context, GenerateRequest) (*GenerateResponse, error)
|
Generate(context.Context, GenerateRequest) (*GenerateResponse, error)
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user