30 Commits

Author SHA1 Message Date
31f2ce3a09 Document Promptkit v0.5.0 2026-08-01 13:18:46 +00:00
fd06e4ca6b Clean up roadmap and fallback profile guidance 2026-08-01 13:16:26 +00:00
e63b8de1e9 Complete application fallback profile implementation 2026-08-01 12:39:01 +00:00
9354d2b373 Add application fallback profile sources 2026-08-01 12:37:01 +00:00
01ca5430bd Move profile composition to the engine facade 2026-08-01 12:31:50 +00:00
ae2179d103 Complete optional parameter omission 2026-08-01 02:43:02 +00:00
a248433d0f Omit unset optional request parameters 2026-08-01 02:41:44 +00:00
bd6cffc9d0 Prepare documentation for Promptkit v0.4.0 2026-07-30 23:48:25 +00:00
e40c4f182b Document structured capacity errors 2026-07-30 23:26:44 +00:00
7428e50c2c Expose structured capacity errors 2026-07-30 23:24:04 +00:00
63c67a4520 Add internal capacity error identity 2026-07-30 23:21:27 +00:00
25a7052a3d Clarify prompt input requirement documentation 2026-07-30 22:10:33 +00:00
fc3255967e Document prompt inspection API 2026-07-30 21:06:33 +00:00
e920168b30 Expose prompt inspection through the engine 2026-07-30 21:03:47 +00:00
272b6a4bc1 Add internal prompt inspection 2026-07-30 21:00:05 +00:00
dde48a31fc Document profile inspection API 2026-07-30 19:57:07 +00:00
242eace4a7 Expose profile inspection through the engine 2026-07-30 19:52:00 +00:00
0bf5f88136 Add internal profile inspection resolution 2026-07-30 19:48:09 +00:00
369ab5392d Add feature roadmap and implementation plan for profile inspection API 2026-07-30 19:42:51 +00:00
2ba0146e5d Tighten prepared execution credential handling 2026-07-30 19:06:17 +00:00
6112c2af0c Document prepared execution workflow 2026-07-30 18:27:03 +00:00
f5e12c00f5 Expose prepared execution handles 2026-07-30 18:20:35 +00:00
49fe402dd2 Add prepared execution lifecycle to the runner 2026-07-30 18:10:28 +00:00
c301eb8d55 Add frozen validation preparation plans 2026-07-30 17:59:45 +00:00
c13e9710d9 Add feature roadmap and implementation plan for downstream consumer wishlist items 2026-07-30 17:54:44 +00:00
87b5ec3d75 Organize downstream feature requests in the future roadmap 2026-07-30 17:13:26 +00:00
cb4028a637 Add feature roadmaps with wishlists from downstream consumers 2026-07-30 16:59:37 +00:00
5a1bff4529 Prepare the local backend convenience release 2026-07-30 04:01:19 +00:00
805a7f965d Document local backend configuration paths 2026-07-30 03:40:52 +00:00
147f5e5ff5 Add local backend convenience constructor 2026-07-30 03:38:23 +00:00
50 changed files with 4733 additions and 1571 deletions

View File

@@ -33,6 +33,15 @@ boundary and constraints that framework work must preserve.
## Release Guidance
Consumers upgrading from `v0.4.0` to `v0.5.0` should read the
[v0.5.0 changelog and migration guide](docs/releases/v0.5.0.md).
Consumers upgrading from `v0.3.0` to `v0.4.0` should read the
[v0.4.0 changelog and adoption guide](docs/releases/v0.4.0.md).
Consumers upgrading from `v0.2.0` to `v0.3.0` should read the
[v0.3.0 changelog](docs/releases/v0.3.0.md).
Consumers moving from `v0.1.0` to `v0.2.0` should read the
[v0.2.0 changelog and migration guide](docs/releases/v0.2.0.md).

View File

@@ -9,6 +9,11 @@ import (
// backend.
const BackendOpenRouter = backend.OpenRouterID
// BackendLocal is the case-sensitive conventional ID used by [LocalBackend].
// It is not a built-in or reserved backend and must be registered with
// [WithBackend].
const BackendLocal = "local"
// Backend configures one engine-scoped OpenAI-compatible backend.
//
// Backend has no stable JSON representation. Use keyed literals so additions
@@ -35,15 +40,35 @@ type Backend struct {
// calls allowed for this backend within one Engine. Zero leaves the backend
// unlimited. A negative value makes NewEngine fail with ErrInvalidConfig.
ConcurrencyLimit int
// QueueCapacity controls how many additional Run calls may be admitted
// beyond ConcurrencyLimit. Nil uses 1024 when ConcurrencyLimit is positive;
// a pointer uses its exact value, including zero. The pointed-to value must
// be non-negative, and QueueCapacity must be nil when ConcurrencyLimit is
// zero. Their sum must fit in an int. WithBackend copies the value and does
// not retain the pointer.
// QueueCapacity controls how many additional Run or RunPrepared calls may
// be admitted beyond ConcurrencyLimit. Nil uses 1024 when ConcurrencyLimit
// is positive; a pointer uses its exact value, including zero. The pointed-to
// value must be non-negative, and QueueCapacity must be nil when
// ConcurrencyLimit is zero. Their sum must fit in an int. WithBackend copies
// the value and does not retain the pointer.
QueueCapacity *int
}
// LocalBackend returns a caller-owned Backend for a conventional local
// OpenAI-compatible endpoint. It sets ID to BackendLocal and copies endpoint
// and concurrencyLimit into Endpoint and ConcurrencyLimit without
// normalization or validation. APIKeyEnv, ExtraParams, and QueueCapacity keep
// their zero values.
//
// LocalBackend does not read environment variables, register the value, or
// mutate engine or package state. Supply the returned value through
// [WithBackend]; [NewEngine] then applies the ordinary backend validation and
// concurrency semantics, including default queue capacity for a positive
// limit, unlimited behavior for zero, and ErrInvalidConfig for a negative
// limit.
func LocalBackend(endpoint string, concurrencyLimit int) Backend {
return Backend{
ID: BackendLocal,
Endpoint: endpoint,
ConcurrencyLimit: concurrencyLimit,
}
}
// WithBackend adds one Backend registration to the constructed Engine.
//
// Registrations accumulate in option order. Every normalized ID must be unique

View File

@@ -60,7 +60,18 @@ func TestEngineRejectsRunBeforeCompletionWhenAdmissionIsFull(t *testing.T) {
awaitCapacitySignal(t, reader.entered, "first artifact read")
result, err := engine.Run(context.Background(), capacityInputRequest("http://second.example/v1"))
canceledContext, cancel := context.WithCancel(context.Background())
cancel()
result, err := engine.Run(canceledContext, capacityInputRequest("http://canceled.example/v1"))
if result != nil || !errors.Is(err, context.Canceled) {
t.Fatalf("canceled capacity admission=(%+v, %v), want context cancellation", result, err)
}
var canceledCapacityErr *promptkit.CapacityError
if errors.Is(err, promptkit.ErrCapacityExceeded) || errors.As(err, &canceledCapacityErr) {
t.Fatalf("canceled admission exposed capacity rejection: %v", err)
}
result, err = engine.Run(context.Background(), capacityInputRequest("http://second.example/v1"))
if result != nil {
t.Fatalf("capacity rejection returned partial result: %+v", result)
}
@@ -70,6 +81,21 @@ func TestEngineRejectsRunBeforeCompletionWhenAdmissionIsFull(t *testing.T) {
if errors.Is(err, promptkit.ErrInvalidRequest) || errors.Is(err, promptkit.ErrLLMGenerate) {
t.Fatalf("capacity rejection had an unrelated category: %v", err)
}
var capacityErr *promptkit.CapacityError
if !errors.As(err, &capacityErr) || capacityErr == nil {
t.Fatalf("capacity rejection=%v, want CapacityError", err)
}
if capacityErr.BackendID != "limited" {
t.Fatalf("capacity backend ID=%q, want limited", capacityErr.BackendID)
}
capacityErr.BackendID = "changed"
result, err = engine.Run(context.Background(), capacityInputRequest("http://third.example/v1"))
var subsequentCapacityErr *promptkit.CapacityError
if result != nil || !errors.As(err, &subsequentCapacityErr) ||
subsequentCapacityErr == nil || subsequentCapacityErr.BackendID != "limited" {
t.Fatalf("subsequent capacity rejection=(%+v, %v), want independent limited CapacityError", result, err)
}
if calls := reader.callCount(); calls != 1 {
t.Fatalf("artifact calls=%d, want only the admitted run", calls)
}
@@ -174,6 +200,19 @@ func TestCapacityExceededSentinelContract(t *testing.T) {
if promptkit.ErrCapacityExceeded == nil {
t.Fatal("ErrCapacityExceeded is nil")
}
var nilCapacityErr *promptkit.CapacityError
zeroCapacityErr := &promptkit.CapacityError{}
populatedCapacityErr := &promptkit.CapacityError{BackendID: "limited"}
for _, capacityErr := range []error{nilCapacityErr, zeroCapacityErr} {
if !errors.Is(capacityErr, promptkit.ErrCapacityExceeded) {
t.Fatalf("capacity error=%v, want ErrCapacityExceeded", capacityErr)
}
}
var discoveredCapacityErr *promptkit.CapacityError
if !errors.As(populatedCapacityErr, &discoveredCapacityErr) || discoveredCapacityErr != populatedCapacityErr {
t.Fatalf("populated capacity error is not discoverable: %v", populatedCapacityErr)
}
for _, unrelated := range []error{
promptkit.ErrInvalidConfig,
promptkit.ErrInvalidRequest,
@@ -181,7 +220,8 @@ func TestCapacityExceededSentinelContract(t *testing.T) {
promptkit.ErrValidation,
} {
if errors.Is(promptkit.ErrCapacityExceeded, unrelated) ||
errors.Is(unrelated, promptkit.ErrCapacityExceeded) {
errors.Is(unrelated, promptkit.ErrCapacityExceeded) ||
errors.Is(populatedCapacityErr, unrelated) {
t.Fatalf("ErrCapacityExceeded aliases unrelated sentinel %v", unrelated)
}
}

40
capacity_error.go Normal file
View File

@@ -0,0 +1,40 @@
package promptkit
import (
"fmt"
"strings"
)
// CapacityError reports bounded admission rejected for a selected backend.
//
// Engine-produced values identify only rejection at Promptkit's bounded
// [Engine.Run] or [Engine.RunPrepared] admission boundary. BackendID is the
// normalized registered backend ID used for routing and capacity; endpoint
// overrides do not change it. Every engine-produced value is nonnil and has a
// nonblank BackendID. Provider errors, active-generation waiting, and caller
// cancellation are not represented by this type.
//
// Callers own returned values and may mutate BackendID without affecting engine
// state or another error. CapacityError and its default Go encoding have no
// stable JSON contract. Consumer-constructed values do not establish that an
// engine rejected work.
type CapacityError struct {
// BackendID is the normalized registered backend ID whose admission was
// rejected.
BackendID string
}
// Error returns diagnostic wording that is not a parsing contract. It is safe
// to call on a nil receiver or a value with a blank BackendID.
func (e *CapacityError) Error() string {
if e == nil || strings.TrimSpace(e.BackendID) == "" {
return ErrCapacityExceeded.Error()
}
return fmt.Sprintf("backend %q admission: %v", e.BackendID, ErrCapacityExceeded)
}
// Unwrap returns ErrCapacityExceeded so errors.Is and errors.As can be used
// together. It is safe to call on a nil receiver or a zero value.
func (e *CapacityError) Unwrap() error {
return ErrCapacityExceeded
}

View File

@@ -170,6 +170,40 @@ func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
}
}
func fromDomainProfileInspection(inspection *domain.ProfileInspection) *ProfileInspection {
if inspection == nil {
return nil
}
return &ProfileInspection{
ProfileID: inspection.ProfileID,
EffectiveModelParams: fromDomainExecutionTarget(inspection.EffectiveModelParams),
APIKeyRequired: inspection.APIKeyRequired,
}
}
func fromDomainPromptInspection(inspection *domain.PromptInspection) *PromptInspection {
if inspection == nil {
return nil
}
inputs := make([]PromptInputDefinition, len(inspection.Inputs))
for i, input := range inspection.Inputs {
inputs[i] = PromptInputDefinition{
Name: input.Name,
Required: input.Required,
ContentType: input.ContentType,
Description: input.Description,
}
}
return &PromptInspection{
PromptID: inspection.PromptID,
PromptVersion: inspection.PromptVersion,
PromptHash: inspection.PromptHash,
DefaultProfileID: inspection.DefaultProfileID,
Inputs: inputs,
OutputContract: fromDomainOutputContract(inspection.OutputContract),
}
}
func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence) ExecutionTargetPresence {
return ExecutionTargetPresence{
Temperature: presence.Temperature,

41
doc.go
View File

@@ -3,23 +3,28 @@
//
// Applications construct an [Engine] with [NewEngine], select filesystem or
// in-memory sources and optional engine-scoped [Backend] registrations, and
// call [Engine.Prepare] or [Engine.Run]. Concrete registries, repositories,
// validators, and the built-in OpenAI-compatible client remain internal
// implementation details.
// call [Engine.InspectPrompt], [Engine.InspectProfile], [Engine.Prepare],
// [Engine.PrepareExecution], [Engine.Run], or [Engine.RunPrepared]. Concrete
// registries, repositories, validators, and the built-in OpenAI-compatible
// client remain internal implementation details.
//
// # Concurrency and ownership
//
// An Engine supports concurrent Prepare and Run calls. Engine-local backend
// policies bound admitted Run calls and model generations where configured,
// while different backend pools and unlimited backends continue independently.
// An injected [LLMClient] or [ArtifactReader] can therefore still receive
// concurrent calls and must be safe for that use.
// An Engine supports concurrent InspectPrompt, InspectProfile, Prepare,
// PrepareExecution, Run, and RunPrepared calls. Engine-local backend policies
// bound admitted Run and RunPrepared calls and model generations where
// configured, while different backend pools and unlimited backends continue
// independently. An injected [LLMClient] or [ArtifactReader] can therefore
// still receive concurrent calls and must be safe for that use.
//
// NewEngine copies in-memory profiles and backend definitions. Prepare and Run
// copy request maps, slices, pointer values, and JSON-compatible extra
// parameters before using them. Returned values and values passed to extension
// interfaces are likewise isolated from engine state. Callers own those copies
// and may mutate them after the call that supplied or returned them.
// NewEngine copies in-memory profiles and backend definitions. Prepare,
// PrepareExecution, and Run copy request maps, slices, pointer values, and
// JSON-compatible extra parameters before using them. InspectPrompt and
// InspectProfile return copied inspection values. Returned values and values
// passed to extension interfaces are likewise isolated from engine state.
// Callers own those copies and may mutate them after the call that supplied or
// returned them. Returned structured errors are likewise caller-owned and may
// be mutated without affecting engine state or another error.
//
// # Security and sensitive data
//
@@ -45,10 +50,12 @@
// [GenerateResponse], [ExecutionTargetPresence], and the string value types
// used by those values.
//
// Construction values, including [Config], [Backend], [RunRequest],
// [ArtifactRef], [ExecutionTargetOverride], [Profile], and
// [OpenAICompatibleProfileConfig], do not have stable JSON representations.
// Direct API keys are nevertheless excluded from JSON for every public value.
// Construction, inspection, handle, and error values, including [Config],
// [Backend], [RunRequest], [ArtifactRef], [ExecutionTargetOverride], [Profile],
// [OpenAICompatibleProfileConfig], [ProfileInspection],
// [PromptInputDefinition], [PromptInspection], [PreparedExecution], and
// [CapacityError], do not have stable JSON representations. Direct API keys
// are nevertheless excluded from JSON for every public value.
//
// JSON timestamps use time.Time's RFC 3339 encoding and are omitted when zero.
// PreparedRun and RunResult durations are encoded as integer milliseconds in

View File

@@ -40,11 +40,67 @@ validation, and default transport behavior. Source discovery, format
validation, and profile precedence are defined by the
[framework format reference](../formats.md).
## Supply Embedded Application Defaults
Use `WithFallbackProfileFS` when an application packages profile definitions
that should apply unless an operator provides an ordinary configured profile
with the same ID. For example, an application can embed its defaults while
continuing to use `ProfileDir` for operator overrides:
```go
//go:embed profiles/*.yaml
var applicationProfiles embed.FS
engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: "prompts",
ProfileDir: operatorProfileDir,
},
promptkit.WithFallbackProfileFS(applicationProfiles, "profiles"),
)
```
Keep application-owned profile IDs and definitions in the embedded source.
Use the ordinary configured profile source for operator overrides. Leave
`operatorProfileDir` empty when the operator did not configure an override
directory; a non-empty path names an authoritative higher-precedence source,
so an unavailable or unreadable directory is an error rather than a reason to
fall back. The
[framework format reference](../formats.md#source-and-profile-precedence)
owns the exact profile format and lookup order; the
[`WithFallbackProfileFS` GoDoc](../../engine.go) owns its option contract and
validation rules.
## Inspect A Prompt Before Preparation
Use [`Engine.InspectPrompt`](../../engine.go) to check one configured prompt's
declared inputs and output workflow without creating placeholder inputs or
resolving a profile:
```go
inspection, err := engine.InspectPrompt(ctx, "meeting.summary", "")
if err != nil {
return err
}
for _, input := range inspection.Inputs {
// Compare the declared input with application configuration.
}
```
Use this configuration-time boundary when the application needs only the
declared prompt interface. Use `InspectProfile` separately when it must also
check a configured profile. Use `Prepare` when it needs inputs, schemas, or
rendered messages, and use prepared execution when that work must remain tied
to later execution. The method's [GoDoc](../../engine.go) owns exact fields,
hash, ownership, and error semantics.
## Prepare Without Model Execution
[`Engine.Prepare`](../../engine.go) resolves the selected prompt and profile,
loads inputs and any structured-output schema, and renders messages without
calling a model client:
calling a model client. Choose it when the prepared value is the final
inspection or persistence result and no later execution must be tied to that
exact snapshot:
```go
prepared, err := engine.Prepare(ctx, promptkit.RunRequest{
@@ -61,12 +117,48 @@ shows a complete runnable setup with a prompt file, in-memory profile, and
inline input. Exact request requirements and prepared-result fields belong to
the [`RunRequest` and `PreparedRun` GoDoc](../../types.go).
## Prepare Now And Execute The Same Snapshot Later
Use [`Engine.PrepareExecution`](../../engine.go) when an application must
inspect or persist preflight details before deciding whether to start model
work, while ensuring that later execution uses those exact rendered messages,
inputs, target settings, and validation resources:
```go
preparedExecution, err := engine.PrepareExecution(ctx, promptkit.RunRequest{
PromptID: "meeting.summary",
Inputs: map[string]promptkit.ArtifactRef{
"note": promptkit.Inline("Synthetic meeting notes"),
},
})
if err != nil {
return err
}
defer preparedExecution.Discard()
details := preparedExecution.Details()
// Inspect or persist an application-selected safe subset of details.
result, err := engine.RunPrepared(ctx, preparedExecution)
```
Preparation does not call the model or reserve backend capacity.
`RunPrepared` executes from the retained snapshot rather than reloading
consumer sources. The handle is opaque in-process state, while `Details`
contains rendered content and remains subject to the application's data
handling policy. The
[`PreparedExecution` and method GoDoc](../../prepared_execution.go) and
[engine operation GoDoc](../../engine.go) own exact lifecycle, engine-binding,
credential, cancellation, timing, and error semantics.
## Execute And Validate
[`Engine.Run`](../../engine.go) performs the same preparation, invokes the
configured model client, classifies the generated artifact, and validates the
content. A completed content check may return `ValidationFailed` in the result;
an operational inability to validate returns an error.
content in one call. Choose it when the application does not need a preflight
boundary tied to the eventual execution. A completed content check may return
`ValidationFailed` in the result; an operational inability to validate returns
an error.
The maintained
[offline execution example](../../examples/go-library/run/main.go) injects a
@@ -90,13 +182,41 @@ replace execution settings or the complete output contract.
The [public value GoDoc](../../types.go) defines nil, empty, zero, replacement,
copy, and credential behavior. The
[framework format reference](../formats.md) defines how those request values
interact with prompt definitions, file-backed profiles, built-ins, schemas,
and framework defaults.
interact with prompt definitions, file-backed and application fallback
profiles, built-ins, schemas, and framework defaults.
For programmatic profiles,
[`OpenAICompatibleProfile`](../../profiles.go) converts ordinary
OpenAI-compatible settings into a value accepted by `WithProfiles`.
### Inspect A Profile Before Prompt Work
Use [`Engine.InspectProfile`](../../engine.go) to validate one configured
profile without constructing a synthetic prompt or placeholder inputs. It
resolves the profile's effective target but does not prepare or execute a
prompt:
```go
inspection, err := engine.InspectProfile(ctx, profileID)
if err != nil {
return err
}
target := inspection.EffectiveModelParams
if target.APIKeyEnv != "" {
// Apply application policy for the named environment variable.
} else if inspection.APIKeyRequired {
// Arrange a direct credential before later execution.
}
```
Use this configuration-time boundary when only the profile and its target need
checking. Use `Prepare` when the application also needs prompt, input, schema,
or rendering work; use prepared execution when that work must remain tied to a
later execution. Inspection reports credential requirements but leaves the
timing of credential enforcement to the application. The method's
[GoDoc](../../engine.go) owns its exact result and error contract.
### Set A Per-Run Session And Reasoning
Supply a direct session ID when one prompt should be correlated with a
@@ -124,35 +244,86 @@ providers. The
[`RunRequest` and `ExecutionTargetOverride` GoDoc](../../types.go) owns the
exact normalization, precedence, error, copying, and exposure contract.
### Register A Custom Backend
### Configure A Local OpenAI-Compatible Endpoint
Register a reusable OpenAI-compatible connection once, then select it from a
profile. This local backend limits model generation to two simultaneous calls;
because `QueueCapacity` is omitted, the engine admits up to 1024 additional
calls waiting behind them:
Choose the smallest configuration that fits how the endpoint will be reused.
#### Use An Endpoint-Only Profile
Put the endpoint directly on an in-memory profile when only that profile needs
it and shared backend identity or capacity policy is unnecessary:
```go
engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: "prompts",
},
promptkit.WithBackend(promptkit.Backend{
ID: "local",
Endpoint: "http://localhost:8000/v1",
APIKeyEnv: "LOCAL_LLM_API_KEY",
ConcurrencyLimit: 2,
promptkit.WithProfiles(promptkit.Profile{
ID: "local-summary",
Endpoint: "http://localhost:8000/v1",
Model: "example-model",
}),
)
```
Endpoint-only profiles have an empty backend ID and remain unrestricted by
backend capacity policy.
#### Use The Conventional Local Backend
Use `LocalBackend` when profiles should share the conventional `local`
identity, endpoint, and concurrency limit:
```go
engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: "prompts",
},
promptkit.WithBackend(
promptkit.LocalBackend("http://localhost:8000/v1", 2),
),
promptkit.WithProfiles(promptkit.Profile{
ID: "local-summary",
BackendID: "local",
BackendID: promptkit.BackendLocal,
Model: "example-model",
}),
)
```
The helper is explicit: it does not pre-register a backend or read environment
variables. Supplying a positive limit leaves queue capacity omitted, so normal
backend registration selects the existing default waiting capacity of 1024.
The returned value still enters the engine through `WithBackend`.
#### Configure A Complete Backend
Use a keyed `Backend` value for authentication, extra request parameters, an
explicit queue capacity, a custom ID, or multiple local endpoints:
```go
noWaiting := 0
engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: "prompts",
},
promptkit.WithBackend(promptkit.Backend{
ID: "local-gpu",
Endpoint: "http://gpu-host:8000/v1",
APIKeyEnv: "LOCAL_GPU_API_KEY",
ExtraParams: map[string]any{"provider_option": "enabled"},
ConcurrencyLimit: 2,
QueueCapacity: &noWaiting,
}),
promptkit.WithProfiles(promptkit.Profile{
ID: "gpu-summary",
BackendID: "local-gpu",
Model: "example-model",
}),
)
```
Use distinct custom IDs when registering multiple local endpoints.
Registrations belong to one engine and custom IDs cannot replace built-ins.
The [`Backend` and `WithBackend` GoDoc](../../backends.go) defines validation,
copying, uniqueness, exact concurrency-field semantics, and request-default
behavior.
The [`Backend`, `LocalBackend`, and `WithBackend` GoDoc](../../backends.go)
defines exact construction, validation, copying, uniqueness, concurrency, and
request-default behavior.
Both file-backed and in-memory profiles select a registration through
`backend` or `Profile.BackendID`. Profile and request endpoint overrides retain
@@ -173,8 +344,8 @@ zero:
```go
noWaiting := 0
backend := promptkit.Backend{
ID: "local",
Endpoint: "http://localhost:8000/v1",
ID: "local-gpu",
Endpoint: "http://gpu-host:8000/v1",
ConcurrencyLimit: 2,
QueueCapacity: &noWaiting,
}
@@ -236,6 +407,11 @@ When a limited backend has admitted all active and waiting calls, handle
```go
result, err := engine.Run(ctx, request)
if errors.Is(err, promptkit.ErrCapacityExceeded) {
var capacityErr *promptkit.CapacityError
if errors.As(err, &capacityErr) {
// Record capacityErr.BackendID using application-owned diagnostics.
}
// Apply application policy: shed work, report overload, or retry later.
}
```
@@ -243,8 +419,9 @@ if errors.Is(err, promptkit.ErrCapacityExceeded) {
A rejected call returns no partial result and does not invoke the model
client. Promptkit does not prescribe retries or map this error to an HTTP
status; those choices remain with the consuming application. The
[`Engine.Run` and error GoDoc](../../engine.go) owns exact error and
cancellation identities.
[`CapacityError` GoDoc](../../capacity_error.go) owns the exact typed-error
contract, while the [`Engine.Run` and error GoDoc](../../engine.go) owns broad
error and cancellation identities.
## Application Boundary

View File

@@ -44,98 +44,3 @@ Start with:
For cross-cutting changes, follow every applicable row. Do not create
placeholder documents for packages, APIs, or integrations that do not yet
exist.
## Maintainer-Run Validation
Promptkit does not currently use hosted CI. Maintainers are responsible for
running the documented checks before accepting changes. Run the default Go
validation from the Promptkit repository root:
```sh
go test ./...
go test -race ./...
go vet ./...
go build ./...
go run ./examples/go-library/prepare
```
Check formatting across every tracked Go file:
```sh
gofmt -l $(git ls-files '*.go')
```
The formatting command must produce no paths. Follow every added or changed
Markdown link and confirm its target exists. Finally, check whitespace:
```sh
git diff --check
```
Documentation-only work does not require unrelated new tests, but it still
requires link validation and `git diff --check`. Run the Go validation whenever
documentation changes commands, examples, generated output, or another
behavior checked by the module.
## Focused Validation
Use focused checks while iterating, then run the complete validation sequence
before accepting the change. The root package supports:
```sh
go test .
go vet .
go build .
```
Filter tests by name without assuming a fixed internal package layout:
```sh
go test ./... -run 'TestName'
```
Replace `TestName` with a useful regular expression. Target only paths that
exist, and consult the internal component overview for their owning
documentation. A filtered or package-specific run does not replace the
complete repository validation.
## Coordinated Work With Scriptorium
Promptkit and Scriptorium must remain independently valid. For temporary local
integration, use either a Go workspace outside both repositories or an
uncommitted replacement in the consuming module.
If the repositories are sibling directories, run the workspace commands from
their parent directory:
```sh
go work init ./promptkit ./scriptorium
go work sync
```
Use the workspace only for coordinated local checks. From the same parent
directory, remove it when finished:
```sh
rm -f go.work go.work.sum
```
Alternatively, from the Scriptorium repository root, temporarily point its
Promptkit dependency at the sibling checkout:
```sh
go mod edit -replace gitea.maximumdirect.net/eric/promptkit=../promptkit
```
After coordinated checks, remove the replacement and reconcile module
metadata:
```sh
go mod edit -dropreplace gitea.maximumdirect.net/eric/promptkit
go mod tidy
```
Never commit `go.work`, `go.work.sum`, or a local filesystem `replace`
directive. Before committing in either repository, inspect its module files and
working tree independently. Published consumer versions must depend on a tagged
Promptkit version, not a workspace, local replacement, or unpublished commit.

View File

@@ -58,6 +58,11 @@ When a request omits a version, the selected prompt ID must identify exactly
one definition. When it supplies a version, the ID and version pair must be
unique.
Exact prompt inspection uses this same configured source, strict decoding,
referenced content-file resolution, and ID/version selection. It reports the
selected definition's declared metadata without changing the prompt format or
executing the definition.
### Inputs
Each `inputs` item has these fields:
@@ -152,7 +157,7 @@ extra_params:
| Field | Required | Meaning |
| --- | --- | --- |
| `id` | yes | Non-empty profile identifier. IDs must be unique within one source. |
| `backend` | unless `endpoint` is present | Backend registry ID. It is trimmed and registry membership is checked when the profile is prepared. |
| `backend` | unless `endpoint` is present | Backend registry ID. It is trimmed and registry membership is checked when the profile is prepared or inspected. |
| `endpoint` | unless `backend` is present | Non-empty OpenAI-compatible base URL, including an API version path when required. When both connection fields are present, this overrides the backend endpoint without changing backend identity. |
| `model` | yes | Non-empty provider model name. |
| `temperature` | no | Number from 0 through 2. |
@@ -183,25 +188,27 @@ they also cannot collide with the standard fields listed in the
Execution settings resolve in this order:
1. framework defaults;
1. the framework timeout baseline;
2. the selected backend, when the profile names one;
3. the selected profile; and
4. request `ExecutionTargetOverride` values.
The framework defaults are:
The framework baseline is:
| Setting | Default |
| --- | --- |
| `temperature` | `0` |
| `max_tokens` | `0` |
| `top_p` | `1` |
| `temperature` | Unspecified and omitted from compatible provider requests unless a profile or runtime override selects it. |
| `max_tokens` | Unspecified and omitted from compatible provider requests unless a profile or runtime override selects it. |
| `top_p` | Unspecified and omitted from compatible provider requests unless a profile or runtime override selects it. |
| `timeout_seconds` | `600` |
Numeric zero in a file or in-memory profile means that the profile does not
replace the framework default. Numeric request overrides use pointers, so an
explicit zero is preserved. In particular, an explicit request
`timeout_seconds` of zero disables the per-generation deadline while leaving
the caller context and transport timeout intact.
Numeric zero in a file or in-memory profile does not select a numeric value.
For `temperature`, `max_tokens`, and `top_p`, it leaves the provider control
unspecified. For `timeout_seconds`, it retains the framework deadline. Numeric
request overrides use pointers, so an explicit zero is retained and sent to
compatible providers. In particular, an explicit request `timeout_seconds` of
zero disables the per-generation deadline while leaving the caller context and
transport timeout intact.
Non-empty profile strings replace backend defaults, and non-empty request
strings replace both. Request reasoning is the exception: a nil
@@ -219,18 +226,25 @@ defines how the effective settings are serialized.
### Source And Profile Precedence
An explicit request profile ID takes precedence over the prompt's
`default_profile`. If neither is present, preparation fails.
`default_profile`. If neither is present, preparation fails. Exact profile
inspection instead takes one explicit profile ID and does not use a prompt
default.
Profile sources resolve matching IDs in this order:
1. in-memory profiles supplied with `WithProfiles`;
2. a profile file, `fs.FS`, or configured profile directory; and
3. embedded built-in profiles.
2. the ordinary configured source selected by a profile file, `fs.FS`, or
configured profile directory;
3. application fallback profiles supplied with `WithFallbackProfileFS`; and
4. embedded built-in profiles.
A higher-precedence source falls back only when the profile is absent. An
invalid matching profile is an error and does not fall back. In-memory
`Profile` values follow the same ranges as YAML profiles. They use
`APIKeyRequired` for request-scoped credentials instead of `api_key_env`.
A profile source supplies a complete definition; definitions and their fields
are not merged across sources. A higher-precedence source falls back only when
the requested profile ID is absent. An invalid matching profile is an error and
does not fall back. In-memory `Profile` values follow the same ranges as YAML
profiles. They use `APIKeyRequired` for request-scoped credentials instead of
`api_key_env`. Preparation and exact profile inspection use this same source
precedence.
## Built-In Profile Catalog
@@ -238,8 +252,8 @@ Every built-in selects the `openrouter` backend. The engine's built-in backend
registry supplies `https://openrouter.ai/api/v1` and the environment-variable
name `OPENROUTER_API_KEY`, so individual profiles contain only model and
generation settings. Built-in profile files do not repeat those connection
values. A custom or in-memory profile with the same profile ID takes
precedence.
values. A configured, application fallback, or in-memory profile with the same
profile ID takes precedence.
| Provider | ID | Model |
| --- | --- | --- |

View File

@@ -53,8 +53,9 @@ never also sent as a session header.
The client conditionally includes:
- `temperature`, `max_tokens`, and `top_p` when non-zero or explicitly
present;
- `temperature`, `max_tokens`, and `top_p` only when selected by a profile or
runtime override, including an explicit runtime zero; they are absent when
unspecified;
- non-empty `service_tier` and effective `reasoning_effort`; an explicitly
disabled reasoning setting is empty and therefore omitted; and
- `response_format` for JSON Schema structured output, including its name,

View File

@@ -29,21 +29,28 @@ One pool owns immutable active and total limits plus mutex-protected admission
count, active count, and ordered waiter list. Pool state exists only for the
lifetime of its engine.
## Bounded Run Admission
## Bounded Execution Admission
The runner asks the manager to admit a run after resolving the prompt, profile,
selected backend, effective execution target, credentials, and output contract,
but before schema loading, artifact loading, or rendering. Admission is
immediate: a limited pool either reserves a slot or returns the internal
`ErrCapacityExceeded` identity. The root facade maps that identity to the
public error without treating it as an invalid request or generation failure.
For ordinary `Run`, the runner asks the manager to admit after resolving the
prompt, profile, selected backend, effective execution target, credentials, and
output contract, but before schema loading, artifact loading, or rendering.
`PrepareExecution` performs no admission. `RunPrepared` claims its handle,
rechecks credential availability, and then asks the manager to admit the
frozen backend before generation.
Admission is immediate: a limited pool either reserves a slot or returns only
the internal `ErrCapacityExceeded` identity. The runner attaches the selected
backend identity at its use-case boundary, and the root facade translates that
typed value without treating it as an invalid request or generation failure.
The total admitted bound is the active-generation limit plus its configured
waiting capacity. The returned release function is idempotent. The runner
defers it as soon as admission succeeds and holds the lease across remaining
preparation, initial generation, validation, every repair attempt, and all
failure or cancellation exits. A repair is part of its original admission and
does not reserve another bounded slot.
defers it as soon as admission succeeds. An ordinary run holds the lease across
remaining preparation, initial generation, validation, every repair attempt,
and all failure or cancellation exits. Prepared execution holds the normal
lease across generation, validation, every internal repair attempt, and all
execution exits. A repair is part of its original admission and does not
reserve another bounded slot.
## FIFO Generation Permits
@@ -90,11 +97,17 @@ unlimited admission. The
FIFO transfer, canceled-waiter removal, grant/cancel races, independent pools,
unlimited calls, passthrough behavior, and panic release.
The [runner tests](../../internal/usecase/runner_test.go) own early admission,
lease lifetime, failure release, and shared initial/repair scheduling. The
The [runner tests](../../internal/usecase/runner_test.go) own ordinary early
admission, lease lifetime, failure release, and shared initial/repair
scheduling. The
[prepared-execution use-case tests](../../internal/usecase/prepared_execution_test.go)
own deferred admission, credential ordering, and prepared-execution lease
release. The
[external package capacity tests](../../capacity_contract_test.go) own the
assembled public-engine behavior for configured limits, capacity errors,
endpoint identity, engine independence, and injected clients. The
[prepared-execution contract tests](../../prepared_execution_contract_test.go)
own the public prepared-capacity boundary. The
[root error-boundary tests](../../errors_internal_test.go) own preservation of
the public generation category and context identity when generation is
canceled.

View File

@@ -43,6 +43,21 @@ without making the model client depend on registry configuration.
The implementation has no retry loop, tool-call support, provider catalog,
inbound HTTP behavior, or durable session store.
## Prepared Generation
For [`RunPrepared`](../../engine.go), the runner supplies the model client with
the target, rendered messages, and structured-output constraint retained by
executable preparation. Execution does not reopen or rerender consumer
sources.
Before backend admission, the runner rechecks that the frozen credential
environment-variable name is available. The handle does not retain the
environment value; the model client resolves the value visible when generation
begins. A direct request key remains in private execution state only until the
claimed execution finishes or an unclaimed handle is discarded. Exact public
ownership and redaction semantics belong to the
[`PreparedExecution` GoDoc](../../prepared_execution.go).
## Failure Categories
The package preserves distinct error identities for invalid client

View File

@@ -11,23 +11,23 @@ contributor workflow and validation.
| Component | Implemented responsibility | References |
| --- | --- | --- |
| Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request and result values, profile construction, extension interfaces, value conversion, redacted formatting, public error mapping, and engine-local assembly. | [Package GoDoc](../../doc.go), [backend API](../../backends.go), [engine assembly](../../engine.go) |
| Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request, result, prompt-inspection, and profile-inspection values, opaque prepared-execution handles, profile construction, extension interfaces, value conversion, redacted formatting, typed capacity errors, public error mapping, and engine-local profile-source assembly including application fallbacks. | [Package GoDoc](../../doc.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine assembly](../../engine.go) |
| `examples/go-library/prepare` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, and `Prepare`. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
| `internal/backend` | Constructs each engine's immutable registry from the built-in OpenRouter definition and consumer additions, validates and defensively copies definitions through the shared JSON-value package, and consumes the LLM-owned OpenAI-compatible reserved request-field rule. | [Backend registry](../../internal/backend/registry.go) |
| `internal/capacity` | Owns engine-local bounded run admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation. | [Domain declarations](../../internal/domain/domain.go) |
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
| `internal/jsonvalue` | Validates and deeply copies JSON-compatible extra-parameter trees while preserving supported concrete value types. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
| `internal/jsonvalue` | Validates and deeply copies JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
| `internal/promptdef` | Loads strictly decoded, validated prompt definitions from filesystem and `fs.FS` sources, including version selection and contained file-backed message content. | [Framework formats](../formats.md), [prompt-definition repository](../../internal/promptdef/filesystem_repository.go) |
| `internal/profile` | Loads strictly decoded, validated execution profiles, including backend selection, from filesystem and `fs.FS` sources and composes repositories with error-preserving fallback. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go) |
| `internal/profile/builtin` | Embeds the built-in profile catalog, whose entries select OpenRouter, and combines it with an optional primary repository. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) |
| `internal/profile/builtin` | Embeds the built-in profile catalog, whose entries select OpenRouter. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) |
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates frozen validation plans for prepared execution. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
| `internal/usecase` | Resolves backend, profile, and request settings and coordinates preparation and execution across internal sources, rendering, artifact loading, generation, validation, and optional repair. | [Internal runner](runner.md) |
| `internal/usecase` | Resolves prompt definitions and hashes, profiles, backends, and targets for exact inspection and request settings for preparation, and coordinates ordinary execution and one-attempt prepared execution across internal sources, rendering, artifact loading, generation, validation, capacity, and optional repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
The root package assembles these internal components without exposing their
representations. Consumers depend only on the root facade.

View File

@@ -28,6 +28,34 @@ constructor does not enable one.
Each invocation carries its state in request, prepared-run, and result values.
The runner has no durable run or session store.
## Shared Prompt Selection
The runner uses one prompt-selection and hashing boundary for ordinary
preparation and exact prompt inspection. Preparation retains its early
request-ID check before direct-session normalization; both operations then use
the configured prompt repository to select one definition, load referenced
message content, and calculate the same prompt hash.
Inspection stops after that structural lookup. It does not parse templates or
touch profile, artifact, schema, renderer, validator, admission, or model
collaborators. The root [`Engine.InspectPrompt`](../../engine.go) GoDoc owns
the public operation's exact contract.
## Shared Profile Selection
The runner uses one profile-selection and target-resolution boundary for
ordinary preparation and exact profile inspection. Preparation first selects a
request profile or a prompt default; inspection begins with its required
explicit profile ID. Both then apply the ordinary source precedence, resolve a
named backend, and construct the effective target from framework, backend, and
profile values.
Inspection stops after the resulting endpoint and model are structurally
validated. It does not check credential availability or perform prompt,
artifact, schema, rendering, admission, or model-client work. The root
[`Engine.InspectProfile`](../../engine.go) GoDoc owns the public operation's
exact contract.
## Shared Preparation Pipeline
`Prepare` and `Run` share one private preparation pipeline split at the point
@@ -124,15 +152,17 @@ validation failures. Wrapping preserves the package identities mapped by the
public facade and retains collaborator identities where they are part of the
internal contract.
Admission capacity exhaustion retains the internal capacity identity and adds
the selected backend ID as context. It is not recategorized as an invalid
request or generation failure, and no partial result is returned. A context
already done at admission retains its context identity directly. Cancellation
while waiting for an active generation permit prevents client invocation when
it wins the grant race; the model-client boundary then preserves the context
error through the generation-failure category. Deferred release restores the
admission lease on preparation, generation, validation, repair, and
cancellation failures.
Admission capacity exhaustion retains the internal capacity identity. At the
use-case boundary, the runner attaches the selected backend ID in an internal
typed error, and the root facade copies that value into the public
[`CapacityError`](../../capacity_error.go) without parsing diagnostic text. It
is not recategorized as an invalid request or generation failure, and no
partial result is returned. A context already done at admission retains its
context identity directly. Cancellation while waiting for an active generation
permit prevents client invocation when it wins the grant race; the model-client
boundary then preserves the context error through the generation-failure
category. Deferred release restores the admission lease on preparation,
generation, validation, repair, and cancellation failures.
Other context cancellation propagates through the invoked collaborator and is
classified by the owning operation.

View File

@@ -16,6 +16,11 @@ validation modes, built-in catalog, and source precedence.
definitions, selects an ID and optional version, and resolves file-backed
message content within the selected operating-system or `fs.FS` source.
Exact prompt inspection performs one point-in-time lookup through that same
repository and validates referenced message content before returning declared
metadata. It does not parse templates or read profile, input, or schema
sources, and it does not retain the definition for a later execution.
Its package tests own prompt selection, strict decoding, definition validation,
duplicate detection, and source containment:
[prompt-definition repository tests](../../internal/promptdef/repository_test.go).
@@ -23,21 +28,30 @@ duplicate detection, and source containment:
## Profiles And Built-Ins
`internal/profile` loads and validates execution profiles from an
operating-system filesystem or an `fs.FS`. It supports a primary repository
with fallback only when the primary reports that a profile is absent. Strict
YAML decoding recognizes the optional `backend` field, trims its value, and
requires a model plus at least one non-blank backend or endpoint. Loading does
not check registry membership because the available registry belongs to the
assembled engine; the runner checks membership during preparation.
operating-system filesystem or an `fs.FS`. Its overlay repository consults the
next repository only when the higher-precedence repository reports that a
profile is absent. Strict YAML decoding recognizes the optional `backend`
field, trims its value, and requires a model plus at least one non-blank
backend or endpoint. Loading does not check registry membership because the
available registry belongs to the assembled engine; the runner checks
membership during preparation and exact profile inspection.
`internal/profile/builtin` embeds the maintained built-in profile catalog and
can place a caller-selected repository ahead of that catalog. Every embedded
profile selects `openrouter` and inherits its endpoint and credential
environment-variable name from the built-in backend registry rather than
repeating those values. Profile behavior is owned by the
[profile repository tests](../../internal/profile/repository_test.go), while
catalog completeness, the backend-selection invariant, duplicate IDs, and
overlay behavior are owned by the
The root engine assembles profile repositories in precedence order: in-memory
profiles, one ordinary configured source, an application fallback source, then
the embedded built-in catalog. An explicit file or `fs.FS` profile source
replaces `Config.ProfileDir` within the ordinary configured-source category.
Exact profile inspection performs one point-in-time lookup through those
profile sources and checks the resolved target without reading prompt, input,
or schema sources. It does not retain that lookup for a later execution.
`internal/profile/builtin` embeds the maintained built-in profile catalog.
Every embedded profile selects `openrouter` and inherits its endpoint and
credential environment-variable name from the built-in backend registry rather
than repeating those values. Profile loading and overlay behavior are owned by
the [profile repository tests](../../internal/profile/repository_test.go),
while catalog completeness, the backend-selection invariant, and duplicate IDs
are owned by the
[built-in repository tests](../../internal/profile/builtin/repository_test.go).
## Ordinary Artifacts
@@ -68,6 +82,22 @@ filesystem or an `fs.FS`. Invalid generated content is returned as a validation
result; inability to load, register, or compile a schema is an operational
error.
For executable preparation, the built-in validators create a frozen validation
plan. None, basic, and JSON modes retain the effective output contract without
source access. JSON Schema mode loads the root document, resolves and compiles
every transitive reference during preparation, and retains the compiled
validator. The provider-facing structured-output metadata uses that same
captured root document.
`PrepareExecution` also completes prompt and profile selection, artifact
loading and hashing, session and message rendering, and target resolution.
`RunPrepared` uses the retained source-derived state and validation plan; it
does not reopen prompt, profile, input, or schema sources and does not rerender
the request. By contrast, ordinary `Prepare` produces a preparation value only:
a later `Run` performs its own source resolution and preparation.
The [validator tests](../../internal/validate/standard_validator_test.go) own
basic, JSON, JSON Schema, source resolution, schema loading, compilation, and
content-failure behavior.
basic, JSON, JSON Schema, source resolution, schema loading, compilation,
frozen-reference behavior, and content-failure behavior. Prepared execution
orchestration is owned by the
[use-case tests](../../internal/usecase/prepared_execution_test.go).

View File

@@ -90,7 +90,7 @@ engine, err := promptkit.NewEngine(
```
See the
[custom-backend consumer guide](../consumers/pkg-promptkit.md#register-a-custom-backend)
[local-endpoint consumer guide](../consumers/pkg-promptkit.md#configure-a-local-openai-compatible-endpoint)
for task-oriented usage. The
[`Backend` and `WithBackend` GoDoc](../../backends.go) owns exact registration,
validation, copying, defaulting, and uniqueness semantics. The

74
docs/releases/v0.3.0.md Normal file
View File

@@ -0,0 +1,74 @@
# Promptkit v0.3.0
This supplemental changelog summarizes the consumer-facing changes from
`v0.2.0` to `v0.3.0`. The annotated `v0.3.0` tag is the authoritative release
record. Exact current contracts belong to the linked GoDoc and durable
documentation.
## Summary
`v0.3.0` adds a concise way to register the common local OpenAI-compatible
backend configuration:
- `BackendLocal` provides the conventional, non-reserved backend ID `"local"`;
and
- `LocalBackend` constructs an ordinary `Backend` from an endpoint and
concurrency limit.
The helper is explicit and additive. It does not pre-register a backend, read
environment variables, select a model, or replace the complete `Backend`
configuration interface.
## Compatibility
Existing `v0.2.0` consumers require no migration. Endpoint-only profiles,
complete custom `Backend` values, the built-in OpenRouter backend, and existing
registrations using the literal ID `"local"` continue to work unchanged.
## Upgrade
Update the module dependency with:
```sh
go get gitea.maximumdirect.net/eric/promptkit@v0.3.0
go mod tidy
```
Run the consuming project's ordinary tests and race-enabled tests after the
upgrade.
## Configure A Local Backend
Register the convenience value through the existing `WithBackend` option and
select it from one or more profiles:
```go
engine, err := promptkit.NewEngine(
promptkit.Config{PromptDir: "prompts"},
promptkit.WithBackend(
promptkit.LocalBackend("http://localhost:8000/v1", 2),
),
promptkit.WithProfiles(promptkit.Profile{
ID: "local-summary",
BackendID: promptkit.BackendLocal,
Model: "example-model",
}),
)
```
Use an endpoint-only profile when shared backend identity and capacity policy
are unnecessary. Continue to use a complete keyed `Backend` value for custom
IDs, authentication, extra request parameters, explicit queue capacity, or
multiple local endpoints.
See the
[local-endpoint consumer guide](../consumers/pkg-promptkit.md#configure-a-local-openai-compatible-endpoint)
for task-oriented configuration choices. The
[`BackendLocal`, `LocalBackend`, and `WithBackend` GoDoc](../../backends.go)
owns their exact construction, registration, validation, and concurrency
semantics.
## Consumer Action
None. Adopt the convenience constructor when it simplifies local endpoint
configuration.

189
docs/releases/v0.4.0.md Normal file
View File

@@ -0,0 +1,189 @@
# Promptkit v0.4.0
This supplemental changelog and adoption guide summarizes the consumer-facing
changes from `v0.3.0` to `v0.4.0`. The annotated `v0.4.0` tag is the
authoritative release record. Exact current contracts belong to the linked
GoDoc and durable documentation.
## Summary
`v0.4.0` adds four complementary capabilities:
- opaque prepared-execution handles for preparing once, inspecting safe
details, and executing the same frozen snapshot;
- exact prompt-definition inspection without profile resolution or execution;
- exact profile inspection without selecting a prompt or checking credential
availability; and
- structured backend identity on engine admission-capacity rejection.
These APIs let consumers perform more precise preflight work and retain useful
operational context without reproducing Promptkit's internal resolution logic.
## Compatibility
The release is additive for `v0.3.0` consumers. Existing uses of `Prepare`,
`Run`, backend registration, endpoint-only profiles, local-backend helpers,
runtime overrides, public JSON values, and error sentinels continue to work
without migration.
Capacity rejection now returns a structured error while continuing to match
`ErrCapacityExceeded` through `errors.Is`. Error-string wording and direct
sentinel equality were not public contracts.
The new inspection values, capacity error, and prepared-execution handle do not
have stable JSON representations. `PreparedExecution.Details` returns the
existing stable `PreparedRun` value.
## Upgrade
Update the module dependency with:
```sh
go get gitea.maximumdirect.net/eric/promptkit@v0.4.0
go mod tidy
```
Run the consuming project's ordinary and race-enabled tests after upgrading.
No source migration is required.
## Prepare Once And Execute The Same Snapshot
Consumers that need to persist preparation details before generation can now
prepare an opaque, engine-bound execution:
```go
prepared, err := engine.PrepareExecution(ctx, request)
if err != nil {
// Handle preparation failure.
}
defer prepared.Discard()
details := prepared.Details()
// Persist a consumer-selected, appropriately protected preparation record.
result, err := engine.RunPrepared(ctx, prepared)
```
Preparation freezes the selected sources, rendered messages, effective
settings, input content, structured-output metadata, and validation resources
needed by execution. `Details` returns a fresh, caller-owned,
credential-redacted `PreparedRun`.
A handle belongs to its creating engine and permits one execution attempt.
`RunPrepared` consumes that attempt on success and on operational failure.
`Discard` is idempotent and releases an unclaimed handle's execution-only
state. Consumers should discard handles they will not execute, particularly
when a direct request API key may be retained privately until claim or
discard.
Prepared execution does not reserve backend admission during preparation.
Credential availability and backend admission are checked when execution
begins. The execution context is independent of the preparation context.
See the
[prepared-execution consumer guide](../consumers/pkg-promptkit.md#prepare-now-and-execute-the-same-snapshot-later),
the [`PreparedExecution` GoDoc](../../prepared_execution.go), and the
[`Engine.PrepareExecution` and `Engine.RunPrepared` GoDoc](../../engine.go)
for the exact lifecycle, ownership, cancellation, capacity, timing, and
failure contracts.
## Inspect A Prompt
`Engine.InspectPrompt` resolves one prompt ID and optional version through the
engine's configured prompt source:
```go
inspection, err := engine.InspectPrompt(ctx, "report.summary", "")
```
The result includes prompt identity, the opaque prompt hash, declared default
profile ID, declared input metadata, and normalized output contract. It
structurally loads the selected definition and referenced message content but
does not resolve a profile, load schemas or artifacts, render templates,
reserve capacity, or contact a model.
Use inspection for exact configuration checks and metadata discovery. Use
`PrepareExecution` rather than relying on a prior inspection when later
execution must freeze one exact source state, because filesystem-backed
inspection is only a point-in-time lookup.
See the
[prompt-inspection consumer guide](../consumers/pkg-promptkit.md#inspect-a-prompt-before-preparation)
and [`Engine.InspectPrompt` GoDoc](../../engine.go) for exact selection,
ownership, and error behavior.
## Inspect A Profile
`Engine.InspectProfile` resolves one explicit profile independently of a
prompt:
```go
inspection, err := engine.InspectProfile(ctx, "report-production")
```
The result includes the resolved effective execution target and whether a
later request must provide a direct credential. Environment-variable names may
be reported, but inspection does not read credential values or require the
named variable to be populated.
Inspection applies the engine's profile source precedence and resolves any
selected backend. It does not load a prompt, render content, reserve capacity,
or contact a model.
See the
[profile-inspection consumer guide](../consumers/pkg-promptkit.md#inspect-a-profile-before-prompt-work)
and [`Engine.InspectProfile` GoDoc](../../engine.go) for the exact resolution,
credential, ownership, and error contracts.
## Identify Capacity-Rejected Backends
Calls rejected at Promptkit's bounded engine admission boundary continue to
match `ErrCapacityExceeded`. Consumers can additionally obtain the selected
registered backend ID without parsing diagnostic text:
```go
result, err := engine.Run(ctx, request)
if errors.Is(err, promptkit.ErrCapacityExceeded) {
var capacityErr *promptkit.CapacityError
if errors.As(err, &capacityErr) {
// Record capacityErr.BackendID using application-owned diagnostics.
}
// Apply application-owned overload or retry policy.
}
```
The structured error applies to `Run` and `RunPrepared` admission rejection.
It does not represent provider throttling, quota exhaustion, cancellation
while waiting for generation capacity, or another model-client failure.
Promptkit does not prescribe retry timing or transport status mapping.
See the
[error-handling consumer guide](../consumers/pkg-promptkit.md#handle-errors),
the [`CapacityError` GoDoc](../../capacity_error.go), and the
[`ErrCapacityExceeded` GoDoc](../../engine.go) for the canonical contracts.
## Public API Additions
The release adds:
- `Engine.PrepareExecution`;
- `Engine.RunPrepared`;
- `PreparedExecution`, including `Details`, `Discard`, `String`, and
`GoString`;
- `Engine.InspectPrompt`;
- `PromptInspection`;
- `PromptInputDefinition`;
- `Engine.InspectProfile`;
- `ProfileInspection`; and
- `CapacityError`.
No public API was removed.
## Consumer Action
None. Existing `v0.3.0` workflows may upgrade without adopting the new APIs.
Consumers that adopt prepared execution should discard unused handles.
Consumers that need backend-specific capacity diagnostics may add an
`errors.As` check while retaining their existing `errors.Is` classification.

125
docs/releases/v0.5.0.md Normal file
View File

@@ -0,0 +1,125 @@
# Promptkit v0.5.0
This supplemental changelog and migration guide summarizes the consumer-facing
changes from `v0.4.0` to `v0.5.0`. The annotated `v0.5.0` tag is the
authoritative release record. Exact current contracts belong to the linked
GoDoc and durable documentation.
## Summary
`v0.5.0` makes provider requests less prescriptive and adds an application
fallback layer for profile definitions:
- unset optional provider controls are omitted from OpenAI-compatible request
bodies instead of being populated with framework values; and
- `WithFallbackProfileFS` lets an application package profile defaults that
operators can override through the existing ordinary profile sources.
These changes let compatible providers apply their own model defaults while
giving applications stable embedded profile IDs without weakening operator
configuration precedence.
## Compatibility
The release adds one public function and removes no public declaration.
Existing source code should continue to compile.
There is one intentional behavior change: when no profile or runtime override
selects `top_p`, Promptkit no longer sends the former framework value of `1`.
It omits `top_p` and lets the provider choose its behavior. Unset
`temperature` and `max_tokens` are likewise omitted. Explicit nonzero profile
values and runtime values—including explicit runtime zero values—retain their
precedence and wire effect.
Consumers that relied on Promptkit always sending `top_p: 1` should add that
value to the relevant profile or runtime override before upgrading. Consumers
that did not rely on the implicit sampling value require no migration.
Application fallback profiles are opt-in. Engines that do not call
`WithFallbackProfileFS` retain the previous profile-source behavior.
## Upgrade
Update the module dependency with:
```sh
go get gitea.maximumdirect.net/eric/promptkit@v0.5.0
go mod tidy
```
Run the consuming project's ordinary and race-enabled tests after upgrading.
If request payloads or model behavior are asserted in fixtures, review them for
the optional-parameter omission described below.
## Omitted Optional Provider Controls
The built-in OpenAI-compatible client now includes `temperature`,
`max_tokens`, and `top_p` only when a profile or runtime override selects the
value. An explicit runtime zero remains present because runtime override
pointers distinguish zero from an unspecified value.
Promptkit's positive generation deadline remains a framework concern and is
not a provider request-body default. Required request fields, session IDs,
structured output, reasoning selection, credentials, and explicit extra
parameters retain their existing behavior.
See the [framework default and precedence reference](../formats.md#defaults-and-overrides),
the [`ExecutionTargetOverride` GoDoc](../../types.go), and the
[OpenAI-compatible request-body contract](../integrations/openai-compatible-chat.md#request-body)
for current details.
## Embedded Application Fallback Profiles
Applications can package ordinary profile YAML in an `fs.FS` and register it
as a fallback source:
```go
//go:embed profiles/*.yaml
var applicationProfiles embed.FS
engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: "prompts",
ProfileDir: operatorProfileDir,
},
promptkit.WithFallbackProfileFS(applicationProfiles, "profiles"),
)
```
Leave `operatorProfileDir` empty when no operator source is configured. A
configured ordinary source is authoritative: a matching definition overrides
the application fallback, while a read or validation failure remains an error
instead of silently reaching a lower layer.
Profile definitions resolve in this order:
1. in-memory profiles supplied with `WithProfiles`;
2. the ordinary configured source selected by `WithProfileFile`,
`WithProfileFS`, or `Config.ProfileDir`;
3. the application source supplied with `WithFallbackProfileFS`; and
4. Promptkit's embedded built-in profiles.
Only an absent profile ID falls through. Sources provide complete profiles and
do not merge fields. Loading remains lazy, and the new source uses the existing
strict profile YAML and credential rules.
See the
[embedded-default consumer guidance](../consumers/pkg-promptkit.md#supply-embedded-application-defaults),
the [`WithFallbackProfileFS` GoDoc](../../engine.go), and the
[profile source reference](../formats.md#source-and-profile-precedence) for
current details.
## Public API Changes
The release adds:
- `WithFallbackProfileFS`.
No public declaration was removed or changed.
## Consumer Action
- Review any workflow that depended on Promptkit's implicit `top_p: 1` and
configure the value explicitly when required.
- Optionally adopt `WithFallbackProfileFS` when an application should package
overridable profile defaults.
- Run consumer tests after updating the module dependency.

View File

@@ -1,236 +0,0 @@
# Backend-Specific Concurrency Management
**Status:** Complete.
## Purpose
This roadmap defines the scope and target end state for engine-local,
backend-specific concurrency management. It records the intended capability,
consumer value, and important policy choices.
This document is planning material, not a description of current behavior.
Current exported contracts remain owned by Go declarations and GoDoc, backend
registration guidance by the
[consumer guide](../consumers/pkg-promptkit.md#register-a-custom-backend), and
implemented orchestration by the
[internal runner document](../internal/runner.md).
## Motivation
Different model backends can sustain very different request loads. A local
network endpoint may need a small concurrency limit, while OpenRouter can
usually accept substantially more simultaneous work. Requiring every consumer
to build its own semaphores and queues would duplicate routing knowledge,
create inconsistent cancellation behavior, and make it easy for one caller to
bypass the intended backend limit.
Promptkit should own this coordination because it already resolves each run to
an engine-scoped backend identity and owns every model-generation call made by
the runner. Consumers should continue submitting ready-to-run requests through
the synchronous API, including concurrently from multiple goroutines, without
implementing their own backend scheduler.
The buffered queue is a safety boundary, not an ordinary throughput
restriction. Its primary purpose is to prevent a bug or unintended submission
loop from creating an unbounded in-memory backlog.
## Scope
The feature will add optional concurrency policy to registered backends and
coordinate `Run` calls against independent per-backend capacity pools.
Each policy has two distinct controls:
- an active-generation limit, which protects the backend from too many
simultaneous model requests; and
- a bounded waiting capacity, which protects the process from admitting an
unbounded backlog.
Concurrency policy belongs to a backend registration. It is not a profile
model parameter and cannot be overridden per run. Profiles select the policy
through their backend ID, while a profile or request endpoint override remains
in the selected backend's pool.
`Prepare` does not call a model and will remain outside concurrency admission.
## Defaults And Configuration
The built-in OpenRouter backend will use:
- an active-generation limit of 16; and
- a waiting capacity of 1024.
The waiting default is intentionally generous. Reaching it should indicate
abnormal submission pressure rather than normal application behavior.
Consumer-registered backends will remain unlimited unless the consumer
configures an active-generation limit. When a consumer enables a limit and
does not specify waiting capacity, the waiting capacity will default to 1024.
Consumers may configure a different bounded capacity, including zero when
they want no admitted backlog beyond the active-limit-sized run set.
The public representation must distinguish an omitted waiting capacity from
an explicit zero.
Endpoint-only profiles have no backend registration from which to obtain
policy and will remain unlimited. A future engine-wide or endpoint-keyed
policy can be considered separately if consumers demonstrate that need.
Invalid limits or capacities will fail engine construction as invalid
configuration. Policy values will be copied into engine-owned immutable state
along with the rest of the backend registration.
## Admission And Execution Behavior
`Run` remains a synchronous, wait-for-result operation. Concurrent callers may
block inside `Run` while waiting for their selected backend, then receive the
ordinary result or error from that invocation.
For a configured pool, the active-generation limit plus the waiting capacity
defines the maximum number of concurrent `Run` invocations that Promptkit will
accept for that backend. A waiting capacity of zero therefore accepts no more
runs than the active limit. Admission is immediate: a call either reserves one
of those bounded slots or receives the capacity error. An accepted run may
then wait internally for active-generation capacity.
For a limited backend, Promptkit will bound the number of accepted runs before
expensive artifact loading, prompt rendering, and large defensive copies where
practical. Lightweight prompt, profile, and backend resolution may occur first
when it is required to identify the correct capacity pool. This pre-admission
resolution must not become a second execution-precedence path with behavior
that can drift from `Prepare`.
An accepted run retains its admission until it completes or fails. Every
actual model-generation call for that run must separately observe the
backend's active-generation limit. This includes:
- the initial generation;
- every output-repair generation; and
- calls made through either the built-in or an injected model client.
Preparation and output validation should not hold an active-generation permit.
A repair remains part of its already-admitted run, but reacquires active
generation capacity so repairs cannot exceed the backend limit. It must not be
rejected merely because new runs filled the waiting queue after its initial
generation.
Within one backend pool, waiting generation calls should be served in FIFO
order, subject to canceled calls being removed. Different backend pools make
progress independently; a saturated local backend must not consume
OpenRouter's active or waiting capacity.
The feature will not promise ordering across backend pools or completion order
among admitted runs.
## Capacity Failure And Cancellation
When a backend's bounded waiting capacity is full, a new `Run` call will fail
promptly rather than waiting outside the bounded admission system. The public
API will expose a recognizable capacity-exhaustion error identity distinct
from invalid configuration, invalid requests, and model-client failures.
Rejected calls return no partial result and do not invoke the model client.
Waiting within the admitted backlog or for active-generation capacity must
honor the caller's context. Cancellation or deadline expiry will:
- stop waiting promptly;
- release any admission or generation capacity held by that invocation;
- preserve the applicable context error identity; and
- avoid invoking the model client if cancellation wins before generation
starts.
Capacity must also be released after preparation, generation, validation,
repair, or collaborator failure. One failed or canceled run must not reduce
the backend's future usable capacity.
Elapsed `Run` timing will include time spent waiting after the call is
accepted. `PreparedRun` timing will continue to describe preparation rather
than queue waiting.
## Engine And Client Boundaries
All pools and queued state belong to one `Engine`. Separate engines do not
share capacity, even when they register the same backend ID or endpoint. The
feature introduces no process-global scheduler.
The engine will apply policy consistently to the built-in model client and an
injected `LLMClient`. Consumers calling their own client outside Promptkit are
outside this boundary. Injected clients remain responsible for their internal
thread safety and cancellation behavior.
Backend policy is keyed by the resolved backend ID rather than endpoint text.
This preserves stable routing when a selected backend's endpoint is overridden
and avoids accidentally combining unrelated registrations that happen to use
the same URL.
## Queue Lifetime And Observability
Admission state is buffered, ephemeral, and in-process. It is not persisted
and has no survival guarantee across engine disposal or process termination.
Promptkit will not introduce background job ownership or require consumers to
start or stop workers.
The initial feature does not require public queue-depth metrics, callbacks, or
inspection APIs. Capacity errors and ordinary call timing provide the
consumer-visible behavior. Operational observability can be added later
without coupling the scheduling mechanism to an application logging or
metrics system.
## Compatibility
Consumer-registered backends and endpoint-only profiles remain unlimited
unless concurrency is explicitly configured, preserving their existing
behavior.
The built-in OpenRouter backend will change from unlimited concurrency to a
limit of 16 with a bounded waiting capacity of 1024. Ordinary synchronous
calls remain unchanged, while unusually high concurrent use may now wait or
return the capacity error. This behavioral change must be identified in the
release notes for the version that publishes it.
Adding backend policy fields and a public capacity error is otherwise
additive. The change will use a pre-`v1` minor release under Promptkit's
[release policy](../release.md#release-model).
## Non-Goals
This scope does not include:
- asynchronous job handles, polling, or detached result delivery;
- durable or cross-process queues;
- persistence or recovery across engine or process shutdown;
- priorities, scheduling weights, or consumer-defined fairness classes;
- automatic retries, backoff, rate-limit interpretation, or provider quota
discovery;
- token-per-minute or request-per-minute rate limiting;
- dynamic reconfiguration after engine construction;
- per-profile or per-run concurrency overrides;
- endpoint-keyed pooling for profiles without a backend ID;
- process-global coordination across engines;
- application worker lifecycle, logging, tracing, or metrics policy; or
- changes to prompt, profile, schema, or model-provider wire formats.
## Target End State
This roadmap reaches its target end state when:
- each engine independently coordinates configured backend capacity;
- the built-in OpenRouter backend allows 16 active generations and up to 1024
waiting runs;
- consumer backends can opt into their own active and waiting limits while
remaining unlimited by default;
- endpoint overrides retain the selected backend's capacity pool and
endpoint-only profiles remain unlimited;
- synchronous `Run` callers wait for and receive their ordinary result;
- admission is bounded before expensive preparation work where practical;
- every initial and repair generation observes the backend's active limit
without serializing preparation or validation;
- a full waiting queue returns a recognizable capacity error without invoking
the model client;
- cancellation and all failure paths promptly release capacity and preserve
context error identity;
- built-in and injected model clients receive the same scheduling behavior;
- pools remain ephemeral, engine-scoped, and independent across backend IDs;
and
- current-state GoDoc, consumer, internal, and release documentation describe
the implemented behavior once it lands.

82
docs/roadmap/deferred.md Normal file
View File

@@ -0,0 +1,82 @@
# Deferred Feature Ideas
## Purpose
This document catalogs feature ideas that remain potentially useful but have
been deliberately postponed. These ideas are not awaiting ordinary selection
from the [future feature catalog](future.md); each has a stated reason to wait
and should be reconsidered only when its trigger becomes relevant.
Deferred entries are not commitments, schedules, active implementation plans,
or descriptions of current behavior. When an entry is reactivated, move it to
`future.md` for evaluation or directly into a focused roadmap after its open
design dependencies have been resolved.
## Deferred Ideas
### Semantic Execution-Target Fingerprints
**Reason for deferral:** A stable digest requires a deliberate semantic-
equality and versioning design. Notarius can safely use conservative source
hashes and a Promptkit release marker today, while Weatherreporter does not
currently reuse LLM-dependent checkpoints.
Promptkit could expose an opaque equality value for a resolved profile and its
effective generation target. This would let checkpointing consumers detect
generation-affecting configuration changes without hashing YAML presentation
or depending on Promptkit's built-in catalog layout.
The digest should change with semantically relevant state such as the resolved
model, endpoint, backend routing identity, request defaults, extra parameters,
profile generation settings, and selected built-in profile semantics. It
should exclude credential values, concurrency and queue policy, source paths,
comments, formatting, and other representation-only changes. Whether a
credential environment-variable name affects equality must be decided
explicitly. The encoding should remain opaque and internally versioned so
Promptkit can deliberately invalidate earlier digests when its resolution
semantics change.
Reconsider this idea when a downstream consumer needs Promptkit-owned
checkpoint equality or when a broader semantic identity design is selected.
### Eager Source Validation
**Reason for deferral:** Exact prompt and profile inspection may already
provide a sufficiently small validation surface. Experience from downstream
adoption should establish whether an engine-wide operation would add enough
value to justify its broader contract.
Promptkit could provide an explicit offline operation that discovers and
structurally validates configured prompt, profile, and schema sources without
model generation. The normal `NewEngine` path would remain lazy.
An eager operation would need coherent handling for duplicate prompt IDs and
versions, strict YAML decoding, referenced content files, profile/backend
membership, schema syntax and transitive references, context cancellation,
and source-specific public errors. Credential declarations must remain
separate from credential values; checking current environment availability,
if supported at all, should be an explicit option and must not expose secrets.
Reconsider this idea after downstream use of `InspectPrompt`,
`InspectProfile`, and fixture-based preparation demonstrates a concrete gap.
### Structured Generation Errors
**Reason for deferral:** Existing `ErrLLMGenerate` classification, preserved
injected-client errors, and prepared execution details currently provide the
necessary failure boundary. A typed error should wait for stronger downstream
demand and a transport-neutral field design.
Promptkit could expose safe structured generation context through
`errors.As` while preserving `errors.Is(err, ErrLLMGenerate)`. Potential
fields include the selected backend ID and model plus an optional HTTP status
when the built-in OpenAI-compatible transport supplies one.
The design must not expose provider response bodies, endpoints, credential
environment names or values, request content, or generated content. It should
not duplicate prompt and profile provenance already available from a prepared
execution, and it must preserve the identity of errors returned by injected
clients. Retry and backoff policy remains a consumer responsibility.
Reconsider this idea when consumers need structured generation diagnostics
beyond the existing sentinel, wrapped client error, and preparation record.

View File

@@ -12,6 +12,9 @@ consumer value, and important scope boundaries. Defer API design,
implementation details, sequencing, and acceptance criteria until an idea is
selected.
Ideas that have been deliberately postponed rather than left available for
ordinary selection belong in the [deferred catalog](deferred.md).
## Using This Catalog
- Add an idea when its purpose and likely value can be stated clearly.
@@ -23,6 +26,8 @@ selected.
- When an idea is selected, move its active planning to a focused roadmap or,
when it requires a durable architectural decision, an ADR. Update
current-state documentation only when implementation lands.
- Move an idea to `deferred.md` when maintainers decide to retain it but wait
for a stated design dependency, demand signal, or reconsideration trigger.
- Remove ideas that are no longer relevant. Retain a rejected idea only when
its rationale is likely to prevent repeated reconsideration.
@@ -33,9 +38,7 @@ consumers.
## Ideas
No ideas are currently cataloged. Backend-specific concurrency management has
been selected for active planning in the
[focused concurrency roadmap](concurrency.md).
No ideas currently await selection.
## Entry Format

View File

@@ -1,841 +0,0 @@
# Backend-Specific Concurrency Management Implementation Plan
**Status:** Complete.
## Purpose
This document is the decision-complete implementation plan for
[backend-specific concurrency management](concurrency.md). It is written for a
coding agent that will implement each stage in order.
The feature roadmap owns the intended capability, consumer value, policy
choices, compatibility decision, and target end state. This document owns the
concrete API, internal representation, scheduling architecture, implementation
sequence, test ownership, documentation updates, and completion gates.
## Implementation Rules
- Complete the stages in order. Keep the repository compiling and the focused
tests passing at every stage boundary.
- Preserve unrelated working-tree changes. In particular, `concurrency.md` and
the removal of its source idea from `future.md` may already be uncommitted
when implementation begins; retain both.
- Follow every policy under `docs/policy/`, the task-specific reading guide in
`docs/development.md`, and the target behavior in `concurrency.md`.
- Keep the public API in the root `promptkit` package and implementation
details under `internal/`. Do not expose scheduler types or create another
public package.
- Use only the Go standard library for scheduling. Do not add a queue,
semaphore, worker-pool, or metrics dependency.
- Preserve synchronous, wait-for-result `Run`, unrestricted `Prepare`,
engine-local state, endpoint-only profiles, backend-selected profiles,
backend identity through endpoint overrides, and injected `LLMClient`
behavior.
- Do not broaden the work into asynchronous jobs, durable queues, retries,
rate limiting, dynamic configuration, priorities, worker lifecycle,
endpoint-keyed pools, or public queue observability.
- Keep all tests deterministic, bounded, offline, and race-safe. Coordinate
concurrent tests with channels and barriers rather than timing assumptions
or live providers.
- Update exact GoDoc with each exported declaration change. Update durable
current-state documents only after the corresponding behavior is
implemented.
- Test configurable mechanisms with small test-owned limits. Assert the exact
OpenRouter `16` and default queue `1024` values only at the registry contract
that owns those operational defaults.
- Do not create a release, change a module version, or tag a commit. The final
implementation handoff must identify the built-in OpenRouter behavior change
for the next pre-`v1` minor release.
## Fixed Design
### Public Backend Configuration
Append these fields to the existing root `Backend` type in `backends.go`:
```go
type Backend struct {
// Existing fields remain unchanged and in their current order.
ConcurrencyLimit int
QueueCapacity *int
}
```
Use these exact semantics:
| Public values | Meaning |
| --- | --- |
| `ConcurrencyLimit == 0`, `QueueCapacity == nil` | Unlimited backend; preserve current behavior. |
| `ConcurrencyLimit > 0`, `QueueCapacity == nil` | Limit active generations and use the default waiting capacity of 1024. |
| `ConcurrencyLimit > 0`, `QueueCapacity != nil` | Limit active generations and use the pointed-to capacity exactly, including zero. |
| `ConcurrencyLimit < 0` | Invalid engine configuration. |
| `QueueCapacity != nil` and `*QueueCapacity < 0` | Invalid engine configuration. |
| `ConcurrencyLimit == 0` and `QueueCapacity != nil` | Invalid engine configuration because a queue without an active limit has no defined consumer value. |
`ConcurrencyLimit` counts simultaneous calls to the engine-owned internal
model-client boundary for this backend. `QueueCapacity` controls additional
accepted `Run` invocations beyond that limit. The maximum admitted runs for a
limited backend is therefore:
```text
ConcurrencyLimit + effective QueueCapacity
```
Guard that addition against integer overflow during backend validation.
Do not impose an arbitrary upper bound beyond non-negativity and overflow
safety.
The `QueueCapacity` pointer exists only to distinguish omission from explicit
zero. `WithBackend` and `NewEngine` must not retain the caller's pointer.
`Backend` continues to have no stable JSON representation, and consumers
remain directed to keyed literals.
Do not add concurrency fields to `Profile`, `ExecutionTarget`,
`ExecutionTargetOverride`, `RunRequest`, prompt or profile files, or stable
prepared/result JSON.
### Built-In And Custom Defaults
The backend registry owns these exact operational defaults:
```go
const (
openRouterConcurrencyLimit = 16
defaultQueueCapacity = 1024
)
```
The built-in `openrouter` definition has a normalized concurrency limit of 16
and queue capacity of 1024.
Consumer registrations remain unlimited when concurrency is omitted. For a
consumer backend with a positive limit and omitted queue capacity, normalize
the queue capacity to 1024. Preserve an explicitly configured zero.
Consumers still cannot replace the reserved `openrouter` registration.
Endpoint-only profiles have no backend policy and remain unlimited. A selected
backend retains its pool when a profile or request overrides only its endpoint.
### Internal Backend Representation
Extend `internal/domain.Backend` with scalar policy values and explicit
presence rather than retaining a pointer:
```go
type Backend struct {
// Existing fields...
ConcurrencyLimit int
QueueCapacity int
QueueCapacitySet bool
}
type BackendCapacityPolicy struct {
ConcurrencyLimit int
QueueCapacity int
}
```
`WithBackend` converts the public pointer into `QueueCapacity` plus
`QueueCapacitySet`. Registry normalization validates the combinations above,
fills the default, and leaves every limited stored backend with
`QueueCapacitySet == true`. Unlimited stored backends retain zero values and
`QueueCapacitySet == false`.
Add this internal registry method:
```go
func (r *Registry) CapacityPolicies() map[string]domain.BackendCapacityPolicy
```
It returns a newly allocated map containing only limited backends. Values are
scalars, so callers cannot mutate registry state. The built-in OpenRouter
policy is included. `GetBackend` continues returning a defensive backend copy,
now including normalized scalar capacity metadata.
Capacity policy is operational registry metadata. Do not merge it into an
execution target or expose it to injected model clients.
### Public Capacity Error
Add this root sentinel beside the other run errors in `engine.go`:
```go
var ErrCapacityExceeded = errors.New("backend capacity exceeded")
```
Its GoDoc must state that it identifies a `Run` rejected because the selected
backend has already admitted `ConcurrencyLimit + QueueCapacity` runs. It is
not an invalid request, an LLM/provider rate-limit response, or an
`ErrLLMGenerate` failure.
The internal capacity component owns a corresponding internal
`ErrCapacityExceeded`. Add its mapping in `publicErrorFor` before the broader
generation and invalid-request cases. The public error must preserve the
internal error through wrapping while matching `ErrCapacityExceeded` with
`errors.Is`.
A capacity rejection returns no partial result and must not invoke the
artifact reader, renderer, schema loader, validator, or model client. Prompt,
profile, and backend loading needed to select the pool may already have
occurred.
### Internal Capacity Component
Add `internal/capacity` as the single owner of engine-local run admission and
active-generation permits.
Use these package-level boundaries:
```go
var ErrCapacityExceeded error
type Manager struct {
// Private immutable pool map.
}
func NewManager(
policies map[string]domain.BackendCapacityPolicy,
) (*Manager, error)
func (m *Manager) Admit(
ctx context.Context,
backendID string,
) (release func(), err error)
func NewClient(m *Manager, next llm.Client) llm.Client
```
`NewManager` copies the supplied map and creates one independent pool per
limited backend. Defensively reject blank IDs, non-positive concurrency
limits, negative queue capacities, or total-capacity overflow even though the
registry normally supplies normalized values. Construction creates no worker
goroutines.
An absent manager, blank backend ID, or ID absent from the policy map is
unlimited:
- `Admit` succeeds with a non-nil no-op release function; and
- the client wrapper calls the next client directly.
For a limited pool, `Admit` is immediate and context-aware:
1. return `ctx.Err()` if the context is already done;
2. under the pool lock, compare admitted runs with
`ConcurrencyLimit + QueueCapacity`;
3. return an error matching internal `ErrCapacityExceeded` when full; or
4. increment admitted runs and return an idempotent release function.
The release function decrements admission exactly once, even if accidentally
called more than once. It does not release an active-generation permit; those
permits have their own lifetime.
### FIFO Generation Permits
`NewClient` returns an internal `llm.Client` wrapper around either the built-in
client or the public-client adapter. It must preserve requests, successful
responses, nil responses, and collaborator error identities exactly.
`next` must be non-nil; `NewEngine` and internal runner construction maintain
that invariant. A nil manager returns `next` unchanged.
For a configured backend ID, the wrapper:
1. acquires one active-generation permit from the matching pool;
2. waits in FIFO order when the active count equals `ConcurrencyLimit`;
3. removes a canceled waiter and returns `ctx.Err()` when cancellation wins
before the permit is granted;
4. invokes the next client only after a permit is granted; and
5. releases the permit with `defer` after every success, nil response,
collaborator error, panic unwinding, or context outcome.
Implement FIFO and cancellation explicitly with a mutex and an ordered waiter
list. A channel used only as a counting semaphore is insufficient because it
does not define FIFO ordering or safe removal of canceled waiters.
Permit grant and cancellation must have one lock-protected linearization
point. If cancellation removes the waiter first, do not invoke the next
client. If grant wins first, invoke the next client with the caller's context;
the next client may then observe cancellation normally. Never lose or
double-release a permit in this race.
Releasing a permit transfers it to the oldest non-canceled waiter before
making it generally available. Different backend pools never share admission
or active counts.
The active wrapper enforces its limit even if an internal caller invokes it
without a run admission lease. Bounded backlog is guaranteed for ordinary
engine `Run` calls by the runner admission path; no public API exposes the
wrapped internal client directly.
### Engine Assembly
In `NewEngine`, after constructing the validated backend registry:
1. obtain `backendRegistry.CapacityPolicies()`;
2. construct one `capacity.Manager`;
3. construct the selected base internal LLM client exactly as today;
4. wrap that base client with `capacity.NewClient`; and
5. pass both the wrapped client and manager-as-admitter to the runner.
Every `NewEngine` call constructs a distinct manager. Do not cache managers,
pools, or policies in package globals. The wrapper must be applied after a
public injected client is adapted to `internal/llm.Client`, so built-in and
injected clients receive identical scheduling behavior.
If `NewManager` reports a defensive configuration error, make `NewEngine`
return an error matching `ErrInvalidConfig`.
`Prepare` does not use the manager. An injected client remains required to be
safe for concurrent calls because different backend pools and unlimited
backends may still invoke it concurrently.
### Shared Two-Phase Preparation
Refactor `internal/usecase.Runner` so `Prepare` and `Run` share one preparation
pipeline with two private phases. Do not duplicate prompt/profile/backend
selection or execution precedence.
The first phase resolves only the state required before admission:
1. validate `PromptID`;
2. normalize the direct session ID;
3. load the prompt definition;
4. hash the original prompt definition at its existing error-order position;
5. select and load the execution profile;
6. resolve the selected backend;
7. resolve and validate the effective execution target and credentials; and
8. resolve the effective output contract without loading its schema.
Return a private state value containing the loaded definition, normalized
direct session, prompt-definition hash, selected profile ID, effective target,
numeric-presence metadata, effective output contract, and preparation start
time. Keep this value private to `internal/usecase`.
The second phase consumes that state and performs:
1. structured-output schema loading;
2. artifact loading and input hashing;
3. message and prompt-session rendering;
4. direct-session application;
5. rendered-prompt hashing; and
6. `PreparedRun` construction and timing.
Preserve every existing precedence rule, error identity, direct-session
template bypass, hash input, selected identity, copy guarantee, and timing
field. Do not reload the prompt, profile, or backend between phases.
`Runner.Prepare` records its start time, runs both phases consecutively, and
never calls admission. Its behavior and error ordering remain unchanged.
`Runner.Run` records its existing run start time, runs the first preparation
phase, and then calls:
```go
release, err := r.admitter.Admit(ctx, effectiveBackendID)
```
Use a narrow use-case-owned interface with the same signature:
```go
type RunAdmitter interface {
Admit(context.Context, string) (func(), error)
}
```
A nil admitter means unlimited behavior for internal constructors and tests.
On successful admission, immediately `defer release()` around the remainder of
the run. Then run the second preparation phase, initial generation,
validation, and all repair attempts.
If admission returns internal `capacity.ErrCapacityExceeded`, add useful
backend context without changing its identity. If it returns `ctx.Err()`,
preserve that identity directly rather than recategorizing it as invalid
request or generation failure.
This refactor intentionally replaces the current literal `Run`-calls-`Prepare`
implementation with shared private phases. Update current-state documentation
to describe one shared pipeline rather than retaining an inaccurate call-graph
claim.
### Generation And Repair Lifetime
The admission lease covers the entire accepted run:
- second-phase preparation;
- initial generation;
- validation;
- every repair; and
- all failure and cancellation exits.
Preparation and validation do not hold an active-generation permit. The
wrapped client acquires a permit only around each actual `Generate` call.
The runner's initial generation already carries the effective backend ID in
`GenerateRequest.Target`. Preserve that value. `RepairRequest.Target` and the
default repairer's generated request must continue carrying the same backend
ID, allowing each repair to reacquire the same pool's active permit.
When testing or constructing `NewRunnerWithRepairer`, pass the same wrapped
client to both the runner and `NewDefaultOutputRepairer`. Do not add capacity
state to `RepairRequest`, `ExecutionTarget`, or public generation values.
A repair remains within its existing admission lease. It waits for a FIFO
active permit but never performs a second bounded admission and therefore
cannot fail merely because later runs filled the admission capacity.
### Error And Cancellation Semantics
The required public outcomes are:
| Situation | Required error identity |
| --- | --- |
| Admission capacity is full | `ErrCapacityExceeded` only; not `ErrInvalidRequest` or `ErrLLMGenerate`. |
| Context is done before admission succeeds | Preserve `ctx.Err()`; do not return capacity exhaustion. |
| Context cancels while waiting for an active permit | Preserve `ctx.Err()` through the existing `ErrLLMGenerate` generation category. |
| Wrapped client fails after permit acquisition | Preserve existing `ErrLLMGenerate` and collaborator identities. |
| Preparation or validation fails after admission | Preserve its existing category and release admission. |
Maintain the existing rule that `Run` returns no partial result on any
operational error. Do not add queue status to errors or results.
`RunResult.Duration` continues to start at runner entry and therefore includes
pre-admission resolution, accepted preparation, and active-permit waiting.
`PreparedRun.DurationMS` continues to cover only its shared preparation phases;
it does not include later generation waiting. Capacity-rejected calls have no
result or timing value.
### Ownership And Concurrency Safety
The registry, capacity policy map, pool map, and per-pool limits are immutable
after engine construction. Only admission counts, active counts, and waiter
lists are mutable and must be protected by the owning pool mutex.
Do not retain public queue pointers, caller request values, contexts, or
generation requests after their call completes. A canceled waiter must be
unlinked so its context and request cannot remain reachable from the pool.
Do not hold a pool mutex while:
- loading or rendering prompts;
- reading artifacts or schemas;
- invoking a model client;
- validating output;
- closing a waiter notification channel if the implementation could re-enter
pool code; or
- calling consumer code.
No scheduler operation may spawn a goroutine whose lifetime outlasts the
calling `Run`. The zero steady-state goroutine count is part of the
in-process/no-worker-lifecycle design.
## Test Ownership
Use this ownership split and avoid repeating the full policy matrix at every
layer:
- `internal/backend/registry_test.go` owns normalization, validation, the exact
OpenRouter policy, the custom default queue, explicit zero, unlimited
omission, and policy-map copying.
- `internal/capacity/manager_test.go` owns admission bounds, idempotent release,
FIFO active permits, cancellation races, capacity recovery, independent
pools, unlimited IDs, and observed peak concurrency.
- `internal/capacity/client_test.go` owns wrapper request/response/error
transparency and the rule that cancellation before grant does not invoke the
next client. Combine these with manager tests if one coherent package test
expresses the behavior more clearly.
- `internal/usecase/runner_test.go` owns two-phase preparation parity, pool
selection, admission before expensive work, admission release across run
exits, `Prepare` bypass, and repair reuse of the admitted backend.
- Root external-package tests own public configuration conversion, assembled
engine-local behavior, endpoint-override routing, injected-client limiting,
and public capacity/context error identities.
- Existing model-client HTTP tests remain unchanged because scheduling does
not alter the OpenAI-compatible wire contract.
Concurrency tests must use test-owned limits such as one or two and
channel-controlled blocking clients. Record observed active and peak counts
under a mutex or atomics. Do not use `time.Sleep` to infer queue state.
Package-internal tests may inspect a waiter list under its mutex through a
small test helper when necessary to establish deterministic FIFO ordering; do
not add production metrics or hooks solely for tests.
Do not add separate tests for trivial scalar copies when registry or assembled
behavior already protects them.
## Stage 1 — Backend Policy And Public Configuration
**Status:** Complete.
### Goal
Add the public and internal backend policy representation, normalize all
configured states, and expose immutable normalized policies without changing
runtime scheduling yet.
### Work
1. Add `ConcurrencyLimit` and `QueueCapacity` to `Backend` in `backends.go`
with exact GoDoc for unlimited, defaulted, explicit-zero, invalid, and
engine-scoped behavior.
2. Convert the public queue pointer into scalar value plus presence in
`WithBackend`; do not retain the pointer.
3. Add the internal backend policy fields and
`BackendCapacityPolicy` to `internal/domain/domain.go`.
4. Add the two registry-owned constants and configure the built-in OpenRouter
definition with 16 and 1024.
5. Extend `normalizeBackend` with the fixed validation, defaulting, explicit
zero, and overflow rules.
6. Add `Registry.CapacityPolicies`, returning only limited policies in a fresh
map.
7. Update existing backend composite literals and assertions only where the
new fields are relevant. Continue using keyed literals.
### Tests
1. Extend the exact built-in registry test with the OpenRouter limit and queue.
2. Add one coherent table covering unlimited omission, default queue,
explicit-zero queue, negative values, queue-without-limit, and total
overflow.
3. Extend the registry copy/normalization test to prove returned policy maps
cannot mutate registry state.
4. Add root coverage only if needed to prove the public pointer/presence
conversion; do not reproduce registry validation cases at the facade.
### Focused Validation
Run:
```sh
gofmt -w backends.go internal/domain/domain.go \
internal/backend/registry.go internal/backend/registry_test.go
go test . ./internal/backend
go vet . ./internal/backend
git diff --check
```
Include another touched Go test file in `gofmt` only if it actually changed.
### Completion Gate
This stage is complete when every public configuration state has one normalized
internal meaning, OpenRouter exposes exactly 16/1024, custom backends remain
unlimited by omission, and no runtime call is scheduled yet.
## Stage 2 — Engine-Local Capacity Manager
**Status:** Complete.
### Goal
Implement and prove the bounded admission mechanism and FIFO active-generation
client wrapper independently of runner orchestration.
### Work
1. Add `internal/capacity/manager.go` with the manager, immutable policy copy,
per-backend pools, internal error, immediate admission, idempotent release,
and FIFO context-aware active permits.
2. Add `internal/capacity/client.go` with the transparent `llm.Client` wrapper.
3. Use mutex-protected waiter state and an ordered list; explicitly resolve
grant-versus-cancel races.
4. Ensure unlimited and independent-pool fast paths avoid queue allocation.
5. Do not start workers, timers, cleanup goroutines, or process-global state.
### Tests
1. Add a compact constructor-validation table for blank IDs, non-positive
limits, negative queues, and total-capacity overflow.
2. With a small configured policy, prove that exactly
`limit + queueCapacity` admissions succeed, the next matches
`ErrCapacityExceeded`, and a release permits another admission.
3. Prove release is idempotent.
4. Drive more blocked client calls than the active limit and assert observed
peak concurrency never exceeds that limit.
5. Prove FIFO order with deterministic queue-entry synchronization.
6. Cancel the first and a middle waiter and prove they are removed, never call
the wrapped client, and do not block later waiters.
7. Exercise the grant/cancel race repeatedly under `go test -race`, asserting
no permit leak or double invocation.
8. Prove different backend IDs proceed independently and blank, unknown, or
nil-manager paths remain unlimited.
9. Prove request values, successful and nil responses, and collaborator errors
pass through unchanged after permit acquisition.
### Focused Validation
Run:
```sh
gofmt -w internal/capacity/manager.go \
internal/capacity/manager_test.go \
internal/capacity/client.go \
internal/capacity/client_test.go
go test ./internal/capacity
go test -race ./internal/capacity
go vet ./internal/capacity
git diff --check
```
If tests are combined into one file, omit the nonexistent file from `gofmt`.
### Completion Gate
This stage is complete when the standalone component enforces relational
admission and active limits, FIFO cancellation is race-safe, separate pools
are independent, and the wrapper is transparent apart from waiting.
## Stage 3 — Shared Preparation And Early Run Admission
**Status:** Complete.
### Goal
Refactor runner preparation into one shared two-phase pipeline and place
bounded admission after backend resolution but before schema, artifact, and
rendering work.
### Work
1. Add the private pre-admission preparation state and split the existing
`Prepare` logic according to the fixed design.
2. Make `Runner.Prepare` call both phases without an admitter.
3. Add the `RunAdmitter` interface and runner field.
4. Update `NewRunner` and `NewRunnerWithRepairer` to accept the optional
admitter; update internal call sites with nil until root assembly is wired.
5. Change `Runner.Run` to use the first phase, admit by effective backend ID,
defer the returned release, and then use the second phase.
6. Preserve all existing error precedence, target resolution, hashes,
metadata, session behavior, and timing.
7. Return capacity and context errors with the fixed identities. Do not invoke
later collaborators after rejection.
### Tests
1. Keep the existing `Run`/`Prepare` parity coverage passing to prove the
shared phases do not drift.
2. Add a fake admitter that records backend IDs and release calls.
3. Prove a backend-selected run admits with the selected ID even when the
endpoint is overridden.
4. Prove an endpoint-only run uses the unlimited/blank identity and that
`Prepare` never calls admission.
5. Reject admission and assert schema, artifact, renderer, validator, repairer,
and LLM collaborators are not invoked.
6. Prove admission is released after one successful run and representative
second-phase, generation, and validation errors. Prefer a small table around
the single `defer` invariant rather than duplicating every error test.
7. Retain direct-session, backend precedence, credential, hashing, and repair
tests unchanged except for constructor arguments.
### Focused Validation
Run:
```sh
gofmt -w internal/usecase/runner.go \
internal/usecase/runner_test.go
go test ./internal/usecase
go test -race ./internal/usecase
go vet ./internal/usecase
git diff --check
```
### Completion Gate
This stage is complete when `Prepare` remains unrestricted, `Run` admits after
one canonical routing phase and before expensive completion work, every exit
releases admission, and existing preparation semantics remain unchanged.
## Stage 4 — Engine Assembly And Public Runtime Contract
**Status:** Complete.
### Goal
Wire one manager into each engine, schedule built-in and injected clients,
expose the capacity error, and prove assembled runtime behavior.
### Work
1. Add public `ErrCapacityExceeded` and its exact GoDoc in `engine.go`.
2. Map internal capacity exhaustion in `errors.go`.
3. Construct the manager from the registry policy snapshot in `NewEngine`.
4. Wrap the selected internal client after built-in or injected-client
selection and pass the manager and wrapped client to the runner.
5. Update `Engine`, `NewEngine`, `Run`, `WithLLMClient`, and `LLMClient` GoDoc
only where concurrency, capacity, or cancellation statements change.
6. Ensure manager-construction errors match `ErrInvalidConfig`.
7. For internal repair coverage, construct the default repairer with the same
wrapped client used by its runner and confirm repair target backend identity
remains intact.
### Tests
1. Add an external-package assembled test with a small custom limit and a
blocking injected client; assert peak generation equals or remains below
the configured limit.
2. With queue capacity zero, block one accepted run before generation and
assert the next matching-backend run returns `ErrCapacityExceeded`, does not
match `ErrInvalidRequest` or `ErrLLMGenerate`, returns no result, and never
reaches expensive collaborators or the client.
3. In the same or another focused workflow, prove an endpoint override remains
in the selected backend's pool.
4. Prove two engines with the same backend ID have independent capacity.
5. Prove an unlimited custom backend and an endpoint-only profile preserve
concurrent behavior.
6. Cancel a call waiting for an active permit; assert it matches both
`context.Canceled` and `ErrLLMGenerate`, never invokes the injected client,
and leaves capacity reusable.
7. Add one internal repair workflow with concurrent runs or controlled permits
showing initial and repair generations never exceed the same backend limit
and repairs do not perform a second admission.
8. Extend the public error sentinel contract test with
`ErrCapacityExceeded`.
Avoid a second HTTP-level concurrency suite: the capacity client tests and one
assembled injected-client workflow already protect the shared wrapper used by
the built-in client.
### Focused Validation
Run:
```sh
gofmt -w engine.go errors.go backends.go \
internal/usecase/runner.go internal/usecase/runner_test.go \
engine_test.go public_contract_test.go
go test . ./internal/backend ./internal/capacity ./internal/usecase
go test -race . ./internal/capacity ./internal/usecase
go vet . ./internal/backend ./internal/capacity ./internal/usecase
git diff --check
```
Add any newly created capacity files to `gofmt` when they changed in this
stage.
### Completion Gate
This stage is complete when every engine has independent pools, limited runs
are bounded and FIFO at generation, endpoint routing is correct, capacity and
context errors are stable, repairs reuse admission, and both client kinds pass
through the same wrapper.
## Stage 5 — Durable Documentation And Final Validation
**Status:** Complete.
### Goal
Move implemented contracts into their durable owners, record compatibility
impact, and validate the complete repository.
### Work
1. Review every changed exported declaration. Ensure GoDoc is the canonical
owner of exact field types, nil/zero semantics, defaulting, error identity,
engine scope, concurrency safety, cancellation, and source compatibility.
2. Update `doc.go` so its concurrency summary acknowledges backend scheduling
while continuing to require injected collaborators to be concurrency-safe.
3. Update `docs/consumers/pkg-promptkit.md` with task-oriented examples for:
- a limited local backend;
- omitted queue capacity selecting 1024;
- explicit zero queue capacity; and
- handling `ErrCapacityExceeded`.
Keep exact field semantics in GoDoc rather than duplicating a full table.
4. Add `docs/internal/capacity.md` as the durable owner of pool lifecycle,
admission, FIFO active permits, cancellation, client wrapping, and test
ownership.
5. Add `internal/capacity` to `docs/internal/overview.md`.
6. Update `docs/policy/architecture.md` to include the implemented component
and root assembly dependency without turning policy into an API reference.
7. Update `docs/internal/runner.md` to describe the shared two-phase
preparation pipeline, early bounded admission, lease lifetime, generation
permits, repairs, capacity failures, and cancellation.
8. Review `docs/formats.md`; add only a concise link or clarification if needed
to explain that endpoint overrides preserve backend capacity identity.
Do not add concurrency fields to YAML.
9. Do not change the OpenAI-compatible integration contract or
`docs/internal/llm.md` unless implementation changes their current
statements; scheduling is outside the provider wire contract and concrete
model-client implementation.
10. Record in the implementation handoff that built-in OpenRouter now limits
active generations to 16 with queue capacity 1024 and that the release
must be a pre-`v1` minor release. Do not edit the release procedure or
create a tag.
11. After every check passes, set `concurrency.md`, this implementation plan,
and each stage status to `Complete`. Do not remove the roadmaps in the
implementation change; lifecycle retirement follows review.
### Full Validation
Run the complete sequence from `docs/development.md`:
```sh
go test ./...
go test -race ./...
go vet ./...
go build ./...
go run ./examples/go-library/prepare
gofmt -l $(git ls-files '*.go')
git diff --check
```
The formatting command must produce no paths. Follow every added or changed
Markdown link and confirm its target and heading exist.
Also inspect:
```sh
git status --short
git diff --stat
git diff
```
Confirm that:
- only intended backend, capacity, runner, facade, test, documentation, and
roadmap files changed;
- no `go.work`, `go.work.sum`, local module replacement, credential,
generated binary, coverage output, or unrelated change was introduced;
- the built-in OpenRouter policy is exactly 16/1024;
- custom and endpoint-only backends remain unlimited by omission;
- explicit queue zero is distinguishable from omission;
- no capacity value enters execution targets, generated requests, stable JSON,
prompt/profile YAML, or provider payloads;
- every engine owns distinct pools with no package-global mutable state;
- every initial and repair generation uses the active permit wrapper;
- capacity and waiter state is released on success, error, panic unwinding,
and cancellation;
- concurrency tests use deterministic coordination rather than sleeps;
- current-state documentation describes implemented behavior rather than
referring readers to the roadmaps; and
- the feature and implementation roadmaps contain no unresolved work marked
complete.
### Completion Gate
The implementation is complete only when every target-end-state item in
`concurrency.md` is implemented, race-enabled tests demonstrate the configured
limits and cancellation safety, durable contracts no longer depend on roadmap
prose, and the OpenRouter compatibility change is clearly reported for the
next minor release.
## Implementation Handoff
Backend-specific capacity management is implemented and has passed the complete
repository validation sequence. The built-in OpenRouter backend now permits 16
active generations and a waiting capacity of 1024. Custom backends remain
unlimited when their limit is omitted, and endpoint-only profiles remain
unlimited.
Publishing this behavior requires a pre-`v1` minor release. Its release notes
must identify that unusually high concurrent OpenRouter use can now wait or
return `ErrCapacityExceeded`. This implementation does not change a module
version or create a tag.
## Open Questions
None. The feature roadmap and this plan fix the public representation,
registry defaults, admission bound, FIFO generation behavior, early-routing
refactor, cancellation races, error identities, engine and repair lifetimes,
test ownership, compatibility treatment, and non-goals required for
implementation.

322
engine.go
View File

@@ -63,9 +63,10 @@ var (
// ErrPromptRender identifies a failure to render prompt messages or the
// session ID from the resolved inputs and variables.
ErrPromptRender = errors.New("failed to render prompt")
// ErrCapacityExceeded identifies a Run rejected because the selected backend
// already admitted ConcurrencyLimit + QueueCapacity calls. It is not an
// invalid request, an LLM or provider rate-limit response, or ErrLLMGenerate.
// ErrCapacityExceeded identifies a Run or RunPrepared rejected because the
// selected backend already admitted ConcurrencyLimit + QueueCapacity calls.
// A [CapacityError] reports the selected backend ID. It is not an invalid
// request, an LLM or provider rate-limit response, or ErrLLMGenerate.
ErrCapacityExceeded = errors.New("backend capacity exceeded")
// ErrLLMGenerate identifies a model-client failure or a nil successful
// response. Errors returned by an injected LLMClient remain available
@@ -77,12 +78,15 @@ var (
ErrValidation = errors.New("failed to validate output")
)
// Engine prepares and runs Promptkit prompt requests.
// Engine inspects prompts and profiles and prepares and runs Promptkit prompt
// requests.
//
// An Engine is safe for concurrent calls to [Engine.Prepare] and [Engine.Run].
// Each Engine owns independent backend-capacity pools that coordinate Run
// admission and model generation. Injected collaborators may still be invoked
// concurrently across different backend pools or for unlimited backends.
// An Engine is safe for concurrent calls to [Engine.InspectPrompt],
// [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
// [Engine.Run], and [Engine.RunPrepared]. Each Engine owns independent
// backend-capacity pools that coordinate Run and RunPrepared admission and
// model generation. Injected collaborators may still be invoked concurrently
// across different backend pools or for unlimited backends.
type Engine struct {
runner *usecase.Runner
}
@@ -94,9 +98,10 @@ type Config struct {
// It is required unless a WithPromptFS or WithPromptFile option supplies the
// prompt source.
PromptDir string
// ProfileDir is an optional directory whose profiles take precedence over
// embedded built-in profiles. An empty value selects only built-ins unless
// profile options are also supplied.
// ProfileDir is an optional ordinary configured source whose profiles take
// precedence over application fallback and embedded built-in profiles. An
// empty value selects the lower-precedence sources unless a profile-source
// option supplies the ordinary source.
ProfileDir string
// SchemaDir is the root for JSON Schema files. An empty value uses the
// current directory. WithSchemaFS or WithSchemaFile replaces this source.
@@ -115,12 +120,12 @@ type Config struct {
// Option customizes engine construction.
//
// NewEngine applies options in argument order and ignores nil options. Within
// each prompt-source, profile-source, in-memory-profile, schema-source,
// model-client, and artifact-reader category, the last non-nil valid option
// replaces earlier options in that category. WithBackend is the additive
// exception: unique registrations accumulate, and a repeated backend ID is an
// error rather than a replacement. An invalid option fails construction even
// if a later option would replace it.
// each prompt-source, ordinary-profile-source, fallback-profile-source,
// in-memory-profile, schema-source, model-client, and artifact-reader
// category, the last non-nil valid option replaces earlier options in that
// category. WithBackend is the additive exception: unique registrations
// accumulate, and a repeated backend ID is an error rather than a replacement.
// An invalid option fails construction even if a later option would replace it.
type Option interface {
apply(*engineOptions) error
}
@@ -132,26 +137,30 @@ func (f optionFunc) apply(options *engineOptions) error {
}
type engineOptions struct {
llmClient llm.Client
artifactReader artifactadapter.Reader
promptDefs promptdef.Repository
profiles profile.Repository
memoryProfiles profile.Repository
backends []domain.Backend
validator validate.Validator
promptSource bool
profileSource bool
memorySource bool
validatorSource bool
artifactSource bool
llmClient llm.Client
artifactReader artifactadapter.Reader
promptDefs promptdef.Repository
profiles profile.Repository
fallbackProfiles profile.Repository
memoryProfiles profile.Repository
backends []domain.Backend
validator validate.Validator
promptSource bool
profileSource bool
fallbackProfileSource bool
memorySource bool
validatorSource bool
artifactSource bool
}
// WithLLMClient replaces the built-in model client used by [Engine.Run].
// WithLLMClient replaces the built-in model client used by [Engine.Run] and
// [Engine.RunPrepared].
//
// A nil client makes NewEngine fail with ErrInvalidConfig. The Engine schedules
// Generate calls according to the selected backend's capacity policy, but the
// client may still be called concurrently across different backend pools or for
// unlimited backends. The client is not used by [Engine.Prepare].
// unlimited backends. The client is not used by [Engine.Prepare] or
// [Engine.PrepareExecution].
func WithLLMClient(client LLMClient) Option {
return optionFunc(func(options *engineOptions) error {
if client == nil {
@@ -218,12 +227,12 @@ func WithPromptFile(path string) Option {
// WithProfileFS loads execution profiles from fsys under root.
//
// Profiles from this source overlay built-in profiles. Profile YAML must use
// api_key_env for environment-based credentials; raw API keys are rejected.
// fsys must be non-nil and root must be non-empty; otherwise NewEngine fails
// with ErrInvalidConfig. This option replaces Config.ProfileDir and earlier
// file or FS profile-source options, but remains below WithProfiles in
// precedence.
// Profiles from this ordinary configured source take precedence over
// application fallback and built-in profiles. Profile YAML must use api_key_env
// for environment-based credentials; raw API keys are rejected. fsys must be
// non-nil and root must be non-empty; otherwise NewEngine fails with
// ErrInvalidConfig. This option replaces Config.ProfileDir and earlier file or
// FS profile-source options, but remains below WithProfiles in precedence.
func WithProfileFS(fsys fs.FS, root string) Option {
return optionFunc(func(options *engineOptions) error {
if fsys == nil {
@@ -240,11 +249,12 @@ func WithProfileFS(fsys fs.FS, root string) Option {
// WithProfileFile loads execution profiles from the single profile file at path.
//
// The profile overlays built-in profiles. Profile YAML must use api_key_env for
// environment-based credentials; raw API keys are rejected. path must name an
// existing non-directory file when NewEngine applies the option. This option
// replaces Config.ProfileDir and earlier file or FS profile-source options,
// but remains below WithProfiles in precedence.
// The profile takes precedence over application fallback and built-in profiles.
// Profile YAML must use api_key_env for environment-based credentials; raw API
// keys are rejected. path must name an existing non-directory file when
// NewEngine applies the option. This option replaces Config.ProfileDir and
// earlier file or FS profile-source options, but remains below WithProfiles in
// precedence.
func WithProfileFile(path string) Option {
return optionFunc(func(options *engineOptions) error {
fsys, root, err := fileSource(path)
@@ -257,8 +267,41 @@ func WithProfileFile(path string) Option {
})
}
// WithFallbackProfileFS supplies application-owned fallback profile
// definitions from fsys under root.
//
// Profile lookup checks, in order, profiles supplied by WithProfiles; the
// ordinary configured source selected by WithProfileFile, WithProfileFS, or
// Config.ProfileDir; this fallback source; and Promptkit's embedded built-in
// profiles. Each source supplies a complete profile definition; profile fields
// are not merged between sources. Only an absent profile ID proceeds to the
// next source. A matching read, parse, duplicate, validation, or credential
// format failure stops resolution.
//
// Files use the ordinary strict profile YAML and api_key_env credential rules.
// Loading and validation are lazy: NewEngine validates this option's arguments
// but does not read profile files. fsys must be non-nil and root must be
// nonblank; otherwise NewEngine returns an error matching ErrInvalidConfig.
// Repeating this option replaces the earlier valid fallback source.
//
// This option controls profile-definition lookup, not provider or generation
// failover.
func WithFallbackProfileFS(fsys fs.FS, root string) Option {
return optionFunc(func(options *engineOptions) error {
if fsys == nil {
return ErrInvalidConfig
}
if strings.TrimSpace(root) == "" {
return ErrInvalidConfig
}
options.fallbackProfiles = profile.NewFSRepository(fsys, root)
options.fallbackProfileSource = true
return nil
})
}
// WithProfiles configures in-memory profiles that take precedence over
// configured profile files and built-in profiles.
// ordinary configured, application fallback, and built-in profiles.
//
// NewEngine validates and copies every profile. IDs must be unique within one
// call. An invalid profile, duplicate ID, or unsupported ExtraParams value
@@ -344,13 +387,7 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
promptDefs = promptdef.NewFilesystemRepository(cfg.PromptDir)
}
profiles := builtin.NewRepositoryWithDirectory(cfg.ProfileDir)
if options.profileSource {
profiles = builtin.NewRepositoryWithPrimary(options.profiles)
}
if options.memorySource {
profiles = profile.NewOverlayRepository(options.memoryProfiles, profiles)
}
profiles := newProfileRepository(cfg.ProfileDir, options)
backendRegistry, err := backend.NewRegistry(options.backends)
if err != nil {
@@ -403,6 +440,26 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
}, nil
}
func newProfileRepository(profileDir string, options engineOptions) profile.Repository {
repository := builtin.NewRepository()
if options.fallbackProfileSource {
repository = profile.NewOverlayRepository(options.fallbackProfiles, repository)
}
if options.profileSource {
repository = profile.NewOverlayRepository(options.profiles, repository)
} else if strings.TrimSpace(profileDir) != "" {
repository = profile.NewOverlayRepository(profile.NewFilesystemRepository(profileDir), repository)
}
if options.memorySource {
repository = profile.NewOverlayRepository(options.memoryProfiles, repository)
}
return repository
}
func fileSource(name string) (fs.FS, string, error) {
cleanName := strings.TrimSpace(name)
if cleanName == "" {
@@ -423,6 +480,93 @@ func fileSource(name string) (fs.FS, string, error) {
return os.DirFS(dir), filepath.ToSlash(base), nil
}
// InspectPrompt resolves one explicit prompt definition without selecting a
// profile or starting execution work.
//
// InspectPrompt requires a nonblank promptID. It passes nonblank promptID and
// promptVersion values unchanged to the engine's ordinary, case-sensitive
// prompt selection. An empty version succeeds only when that source has one
// selected ID; a nonempty version selects one exact ID/version pair. The
// configured prompt source is used without merging, fallback, or enumeration.
//
// A successful result proves that the selected definition and any referenced
// message content files were structurally loaded. Inputs are returned in
// definition order. DefaultProfileID is declared metadata only and is not
// resolved. OutputContract is the normalized declared contract, with a JSON
// Schema path when declared but without loading or compiling that schema.
// PromptHash is the same opaque equality value as PreparedRun.PromptHash for
// the selected definition and observed source state; its spelling, length,
// encoding, algorithm, and security properties are not contracts.
//
// This method does not return prompt bodies, templates, source paths, schemas,
// rendered messages, or execution settings. It does not resolve a profile or
// credential, read artifacts or schemas, render, validate, admit capacity,
// contact a provider, or generate model output. The returned PromptInspection
// and its input slice are caller-owned. Filesystem-backed inspection is a
// point-in-time lookup and does not freeze a definition for later execution.
//
// A nil Engine returns an error matching ErrInvalidConfig. A blank prompt ID
// matches ErrInvalidRequest. An absent exact ID or version matches
// ErrPromptNotFound and not ErrPromptLoad. Malformed, unreadable, duplicate,
// ambiguous, referenced-content, or hashing failures match ErrPromptLoad.
// Cancellation during lookup matches ErrPromptLoad while preserving the
// context error. InspectPrompt returns no partial result on error.
func (e *Engine) InspectPrompt(
ctx context.Context,
promptID string,
promptVersion string,
) (*PromptInspection, error) {
if e == nil || e.runner == nil {
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
}
inspection, err := e.runner.InspectPrompt(ctx, promptID, promptVersion)
if err != nil {
return nil, mapPublicError(err)
}
return fromDomainPromptInspection(inspection), nil
}
// InspectProfile resolves one explicit profile without selecting a prompt or
// starting execution work.
//
// InspectProfile trims surrounding whitespace from profileID and looks up the
// resulting nonblank ID exactly and case-sensitively through the engine's
// in-memory, ordinary configured-source, application fallback, and built-in
// profile precedence. It applies the framework timeout baseline, selected
// backend, and then selected profile to EffectiveModelParams without a request
// override. BackendID is empty for an endpoint-only profile.
//
// APIKeyEnv in the returned target is an environment-variable name, never its
// value. APIKeyRequired instead reports a direct credential requirement and is
// mutually exclusive with a nonblank APIKeyEnv. InspectProfile neither derives
// an ID from a prompt default_profile nor checks credential availability, so an
// absent or blank named environment variable is not an error.
//
// The returned ProfileInspection and all nested mutable values are
// caller-owned. Filesystem-backed inspection is a point-in-time lookup and
// does not freeze the profile for a later execution. This method does not load
// a prompt, render, read artifacts or schemas, admit backend capacity, contact
// a provider, or generate model output.
//
// A nil Engine returns an error matching ErrInvalidConfig. A blank profile ID
// matches ErrInvalidRequest. An absent exact ID matches ErrProfileNotFound and
// not ErrProfileLoad. Malformed or unreadable profile data, an unknown backend,
// or an invalid resolved target matches ErrProfileLoad. Cancellation during
// profile loading matches ErrProfileLoad while preserving the context error.
// InspectProfile returns no partial result on error.
func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*ProfileInspection, error) {
if e == nil || e.runner == nil {
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
}
inspection, err := e.runner.InspectProfile(ctx, profileID)
if err != nil {
return nil, mapPublicError(err)
}
return fromDomainProfileInspection(inspection), nil
}
// Prepare resolves and renders a prompt request without calling an LLM.
//
// Prepare selects the prompt and profile, resolves any selected backend and
@@ -457,6 +601,38 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err
return fromDomainPreparedRun(prepared), nil
}
// PrepareExecution completely prepares a prompt request without calling the
// configured LLMClient or reserving backend admission capacity.
//
// The returned opaque handle is bound to this Engine and permits one
// [Engine.RunPrepared] invocation. Preparation freezes the selected sources,
// rendered messages, effective settings, inputs, provider structured-output
// metadata, and validation resources needed by that invocation. The handle
// retains a direct RunRequest.APIKey only in private execution state;
// [PreparedExecution.Details] is credential-redacted.
//
// The context governs preparation only. Cancellation after this method
// returns does not invalidate the handle or propagate to RunPrepared.
// PrepareExecution returns the same error categories as [Engine.Prepare] and
// returns no handle on error. A nil Engine returns an error matching
// ErrInvalidConfig.
func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*PreparedExecution, error) {
if e == nil || e.runner == nil {
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
}
domainReq, err := toDomainRunRequest(req)
if err != nil {
return nil, fmt.Errorf("%w: %v", ErrInvalidRequest, err)
}
prepared, err := e.runner.PrepareExecution(ctx, domainReq)
if err != nil {
return nil, mapPublicError(err)
}
return &PreparedExecution{internal: prepared}, nil
}
// Run prepares a request, invokes the configured LLMClient, and validates the
// generated output.
//
@@ -467,9 +643,10 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err
// single-pass even when OutputContract.RepairAttempts is positive.
//
// Run can return every error category documented by [Engine.Prepare], plus
// ErrCapacityExceeded and ErrLLMGenerate. ErrCapacityExceeded identifies
// rejection before artifacts, schemas, rendering, or model generation because
// the selected backend's admission capacity is full; it does not match
// ErrCapacityExceeded and ErrLLMGenerate. An engine admission rejection is
// discoverable as [CapacityError] and still matches ErrCapacityExceeded. It
// occurs before artifacts, schemas, rendering, or model generation because the
// selected backend's admission capacity is full; it does not match
// ErrInvalidRequest or ErrLLMGenerate. Errors from injected clients remain
// available through errors.Is. Cancellation while waiting for model-generation
// capacity matches both ErrLLMGenerate and the context error. Cancellation
@@ -491,3 +668,42 @@ func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
}
return fromDomainRunResult(result), nil
}
// RunPrepared atomically claims and executes a handle created by
// [Engine.PrepareExecution].
//
// A valid owning-Engine invocation consumes the handle's one attempt before
// credential revalidation, backend admission, generation, or validation.
// Cancellation, capacity rejection, generation failure, operational
// validation failure, and success all leave the handle unusable. A nil,
// zero-value, foreign-Engine, discarded, claimed, or used handle returns an
// error matching ErrInvalidRequest; a nil Engine returns ErrInvalidConfig and
// does not claim the handle.
//
// The supplied context governs this execution attempt independently of the
// preparation context. It covers credential revalidation, admission,
// generation, validation, and any internal repair. Result timing begins after
// the claim and excludes preparation and consumer-held delay.
//
// RunPrepared can return ErrInvalidRequest, ErrAPIKeyEnvMissing,
// ErrCapacityExceeded, ErrLLMGenerate, or ErrValidation as applicable while
// preserving documented collaborator and context identities. An engine
// admission rejection is discoverable as [CapacityError] and still matches
// ErrCapacityExceeded. A completed content-validation rejection is returned
// in RunResult, not as an operational error. An operational error returns no
// partial RunResult.
func (e *Engine) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*RunResult, error) {
if e == nil || e.runner == nil {
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
}
var internal *usecase.PreparedExecution
if prepared != nil {
internal = prepared.internal
}
result, err := e.runner.RunPrepared(ctx, internal)
if err != nil {
return nil, mapPublicError(err)
}
return fromDomainRunResult(result), nil
}

View File

@@ -360,14 +360,14 @@ func TestEngineExecutionSettingPrecedence(t *testing.T) {
wantPresence promptkit.ExecutionTargetPresence
}{
{
name: "framework defaults fill zero-valued profile settings",
name: "unspecified provider controls retain framework timeout",
profile: defaultsProfile,
want: promptkit.ExecutionTarget{
Endpoint: defaultsProfile.endpoint,
Model: defaultsProfile.model,
Temperature: 0,
MaxTokens: 0,
TopP: 1,
TopP: 0,
TimeoutSeconds: 600,
ServiceTier: defaultsProfile.serviceTier,
ReasoningEffort: defaultsProfile.reasoningEffort,
@@ -2301,6 +2301,8 @@ func TestSourceOptionsRejectInvalidInputs(t *testing.T) {
{name: "profile fs nil", opt: promptkit.WithProfileFS(nil, "profiles")},
{name: "profile fs empty root", opt: promptkit.WithProfileFS(fstest.MapFS{}, "")},
{name: "profile file empty", opt: promptkit.WithProfileFile("")},
{name: "fallback profile fs nil", opt: promptkit.WithFallbackProfileFS(nil, "profiles")},
{name: "fallback profile fs empty root", opt: promptkit.WithFallbackProfileFS(fstest.MapFS{}, "")},
{name: "schema fs nil", opt: promptkit.WithSchemaFS(nil, "schemas")},
{name: "schema fs empty root", opt: promptkit.WithSchemaFS(fstest.MapFS{}, "")},
{name: "schema file empty", opt: promptkit.WithSchemaFile("")},

View File

@@ -3,6 +3,7 @@ package promptkit
import (
"errors"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
@@ -14,6 +15,11 @@ func mapPublicError(err error) error {
if err == nil {
return nil
}
var internalCapacityError *usecase.CapacityError
if errors.As(err, &internalCapacityError) && internalCapacityError != nil &&
strings.TrimSpace(internalCapacityError.BackendID) != "" {
return &CapacityError{BackendID: internalCapacityError.BackendID}
}
publicErr := publicErrorFor(err)
if publicErr == nil {
return err

View File

@@ -20,3 +20,31 @@ func TestMapPublicErrorPreservesGenerationCancellation(t *testing.T) {
t.Fatalf("mapped error=%v, want context.Canceled", err)
}
}
func TestMapPublicErrorTranslatesCapacityError(t *testing.T) {
internalErr := &usecase.CapacityError{BackendID: "limited"}
err := mapPublicError(internalErr)
var publicErr *CapacityError
if !errors.As(err, &publicErr) || publicErr == nil {
t.Fatalf("mapped error=%v, want public CapacityError", err)
}
if publicErr.BackendID != "limited" {
t.Fatalf("mapped backend ID=%q, want limited", publicErr.BackendID)
}
if !errors.Is(err, ErrCapacityExceeded) {
t.Fatalf("mapped error=%v, want ErrCapacityExceeded", err)
}
if errors.Is(err, ErrInvalidRequest) || errors.Is(err, ErrLLMGenerate) {
t.Fatalf("mapped capacity error has an unrelated category: %v", err)
}
var leakedInternalErr *usecase.CapacityError
if errors.As(err, &leakedInternalErr) {
t.Fatalf("mapped error exposes internal CapacityError: %v", err)
}
internalErr.BackendID = "changed"
if publicErr.BackendID != "limited" {
t.Fatalf("mapped backend ID changed with source error: %q", publicErr.BackendID)
}
}

View File

@@ -124,6 +124,11 @@ func TestManagerAdmissionHonorsContextAndUnlimitedBackends(t *testing.T) {
if err != nil {
t.Fatalf("construct manager: %v", err)
}
release, err := manager.Admit(context.Background(), "limited")
if err != nil {
t.Fatalf("fill limited pool: %v", err)
}
defer release()
ctx, cancel := context.WithCancel(context.Background())
cancel()

View File

@@ -14,9 +14,6 @@ const (
ContentTypeApplicationJSON = "application/json"
OpenAIChatCompletionsPath = "/chat/completions"
ExecutionDefaultTemperature = 0.0
ExecutionDefaultMaxTokens = 0
ExecutionDefaultTopP = 1.0
ExecutionDefaultTimeoutSeconds = 600
)
@@ -26,9 +23,6 @@ var (
func ExecutionTargetDefault() domain.ExecutionTarget {
return domain.ExecutionTarget{
Temperature: ExecutionDefaultTemperature,
MaxTokens: ExecutionDefaultMaxTokens,
TopP: ExecutionDefaultTopP,
TimeoutSeconds: ExecutionDefaultTimeoutSeconds,
}
}

View File

@@ -145,6 +145,16 @@ type PromptDefinition struct {
Validation OutputContract `yaml:"validation"`
}
// PromptInspection is the resolved result of exact prompt inspection.
type PromptInspection struct {
PromptID string
PromptVersion string
PromptHash string
DefaultProfileID string
Inputs []PromptInput
OutputContract OutputContract
}
// PromptInput describes one named input expected by a prompt definition.
type PromptInput struct {
Name string `yaml:"name"`
@@ -236,6 +246,13 @@ type ExecutionTarget struct {
ExtraParams map[string]any `yaml:"extra_params" json:"extra_params"`
}
// ProfileInspection is the resolved result of exact profile inspection.
type ProfileInspection struct {
ProfileID string
EffectiveModelParams ExecutionTarget
APIKeyRequired bool
}
// OutputContract defines the requirements for the output artifact.
type OutputContract struct {
Format OutputFormat `yaml:"format"`

View File

@@ -1,5 +1,5 @@
// Package jsonvalue validates and defensively copies JSON-compatible value
// trees used by public configuration and request boundaries.
// trees used by configuration, request, and prepared-state boundaries.
package jsonvalue
import (
@@ -18,13 +18,19 @@ type visit struct {
ptr uintptr
}
// Copy validates and deeply copies a JSON-compatible value while preserving
// compatible concrete map, slice, array, scalar, and number types.
func Copy(src any) (any, error) {
return copyValue(reflect.ValueOf(src), "value", make(map[visit]struct{}), true)
}
// CopyMap validates and deeply copies an extra-parameter map while preserving
// compatible concrete map, slice, array, scalar, and number types.
func CopyMap(src map[string]any) (map[string]any, error) {
if src == nil {
return nil, nil
}
copied, err := copyValue(reflect.ValueOf(src), "extra_params", make(map[visit]struct{}))
copied, err := copyValue(reflect.ValueOf(src), "extra_params", make(map[visit]struct{}), false)
if err != nil {
return nil, err
}
@@ -35,7 +41,12 @@ func CopyMap(src map[string]any) (map[string]any, error) {
return out, nil
}
func copyValue(value reflect.Value, path string, seen map[visit]struct{}) (any, error) {
func copyValue(
value reflect.Value,
path string,
seen map[visit]struct{},
allowEmptyMapKeys bool,
) (any, error) {
if !value.IsValid() {
return nil, nil
}
@@ -43,7 +54,7 @@ func copyValue(value reflect.Value, path string, seen map[visit]struct{}) (any,
if value.IsNil() {
return nil, nil
}
return copyValue(value.Elem(), path, seen)
return copyValue(value.Elem(), path, seen, allowEmptyMapKeys)
}
if !value.CanInterface() {
return nil, fmt.Errorf("%s: value cannot be copied", path)
@@ -88,22 +99,27 @@ func copyValue(value reflect.Value, path string, seen map[visit]struct{}) (any,
}
seen[current] = struct{}{}
defer delete(seen, current)
return copyValue(value.Elem(), path, seen)
return copyValue(value.Elem(), path, seen, allowEmptyMapKeys)
case reflect.Map:
return copyMapValue(value, path, seen)
return copyMapValue(value, path, seen, allowEmptyMapKeys)
case reflect.Slice:
if value.IsNil() {
return nil, nil
}
return copySequenceValue(value, path, seen)
return copySequenceValue(value, path, seen, allowEmptyMapKeys)
case reflect.Array:
return copySequenceValue(value, path, seen)
return copySequenceValue(value, path, seen, allowEmptyMapKeys)
default:
return nil, fmt.Errorf("%s: unsupported JSON value type %s", path, value.Type())
}
}
func copyMapValue(value reflect.Value, path string, seen map[visit]struct{}) (any, error) {
func copyMapValue(
value reflect.Value,
path string,
seen map[visit]struct{},
allowEmptyMapKeys bool,
) (any, error) {
if value.IsNil() {
return nil, nil
}
@@ -133,10 +149,10 @@ func copyMapValue(value reflect.Value, path string, seen map[visit]struct{}) (an
elementType := value.Type().Elem()
for _, key := range keys {
name := key.String()
if name == "" {
if name == "" && !allowEmptyMapKeys {
return nil, fmt.Errorf("%s: map key must not be empty", path)
}
copied, err := copyValue(value.MapIndex(key), path+"."+name, seen)
copied, err := copyValue(value.MapIndex(key), path+"."+name, seen, allowEmptyMapKeys)
if err != nil {
return nil, err
}
@@ -171,7 +187,12 @@ func copyMapValue(value reflect.Value, path string, seen map[visit]struct{}) (an
return out, nil
}
func copySequenceValue(value reflect.Value, path string, seen map[visit]struct{}) (any, error) {
func copySequenceValue(
value reflect.Value,
path string,
seen map[visit]struct{},
allowEmptyMapKeys bool,
) (any, error) {
var current visit
if value.Kind() == reflect.Slice {
current = visit{typ: value.Type(), ptr: value.Pointer()}
@@ -186,7 +207,12 @@ func copySequenceValue(value reflect.Value, path string, seen map[visit]struct{}
preserveType := true
elementType := value.Type().Elem()
for i := 0; i < value.Len(); i++ {
copied, err := copyValue(value.Index(i), fmt.Sprintf("%s[%d]", path, i), seen)
copied, err := copyValue(
value.Index(i),
fmt.Sprintf("%s[%d]", path, i),
seen,
allowEmptyMapKeys,
)
if err != nil {
return nil, err
}

View File

@@ -44,6 +44,21 @@ func TestCopyMapPreservesTypesAndIsolatesMutations(t *testing.T) {
}
}
func TestCopyAllowsEmptyObjectKeysAndIsolatesMutations(t *testing.T) {
nested := map[string]any{"": []any{"original"}}
copiedValue, err := jsonvalue.Copy(nested)
if err != nil {
t.Fatalf("copy value: %v", err)
}
nested[""].([]any)[0] = "changed"
copied := copiedValue.(map[string]any)
if got := copied[""].([]any)[0]; got != "original" {
t.Fatalf("copied value was not isolated: %v", got)
}
}
func TestCopyMapRejectsInvalidValues(t *testing.T) {
cyclicMap := map[string]any{}
cyclicMap["self"] = cyclicMap

View File

@@ -2,7 +2,6 @@ package builtin
import (
"embed"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
)
@@ -15,17 +14,3 @@ var assets embed.FS
func NewRepository() profile.Repository {
return profile.NewFSRepository(assets, assetRoot)
}
func NewRepositoryWithPrimary(primary profile.Repository) profile.Repository {
if primary == nil {
return NewRepository()
}
return profile.NewOverlayRepository(primary, NewRepository())
}
func NewRepositoryWithDirectory(dir string) profile.Repository {
if strings.TrimSpace(dir) == "" {
return NewRepository()
}
return NewRepositoryWithPrimary(profile.NewFilesystemRepository(dir))
}

View File

@@ -2,14 +2,11 @@ package builtin
import (
"context"
"errors"
"io/fs"
"strings"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
"gopkg.in/yaml.v3"
)
@@ -91,53 +88,3 @@ func loadBuiltInProfileIDs(t *testing.T) map[string]string {
}
return ids
}
func TestRepositoryWithPrimaryUsesPrimaryBeforeBuiltIns(t *testing.T) {
repo := NewRepositoryWithPrimary(staticProfileRepo{
profiles: map[string]string{"mistral-small-3": "custom-model"},
})
p, err := repo.GetProfile(context.Background(), "mistral-small-3")
if err != nil {
t.Fatalf("expected profile to load, got %v", err)
}
if p.Model != "custom-model" {
t.Fatalf("expected primary profile to override built-in, got %+v", p)
}
}
func TestRepositoryWithPrimaryFallsBackToBuiltIns(t *testing.T) {
repo := NewRepositoryWithPrimary(staticProfileRepo{})
p, err := repo.GetProfile(context.Background(), "mistral-small-3")
if err != nil {
t.Fatalf("expected built-in profile to load, got %v", err)
}
if p.ID != "mistral-small-3" {
t.Fatalf("unexpected profile: %+v", p)
}
}
func TestRepositoryWithPrimaryDoesNotFallBackAfterPrimaryError(t *testing.T) {
repo := NewRepositoryWithPrimary(staticProfileRepo{err: profile.ErrInvalidProfile})
_, err := repo.GetProfile(context.Background(), "mistral-small-3")
if !errors.Is(err, profile.ErrInvalidProfile) {
t.Fatalf("expected primary error, got %v", err)
}
}
type staticProfileRepo struct {
profiles map[string]string
err error
}
func (r staticProfileRepo) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) {
if r.err != nil {
return nil, r.err
}
if model, ok := r.profiles[id]; ok {
return &domain.ExecutionProfile{ID: id, Endpoint: "http://primary/v1", Model: model}, nil
}
return nil, profile.ErrProfileNotFound
}

View File

@@ -0,0 +1,24 @@
package usecase
import (
"fmt"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
)
// CapacityError identifies bounded admission rejected for one selected backend.
type CapacityError struct {
BackendID string
}
func (e *CapacityError) Error() string {
if e == nil || strings.TrimSpace(e.BackendID) == "" {
return capacity.ErrCapacityExceeded.Error()
}
return fmt.Sprintf("backend %q admission: %v", e.BackendID, capacity.ErrCapacityExceeded)
}
func (e *CapacityError) Unwrap() error {
return capacity.ErrCapacityExceeded
}

View File

@@ -0,0 +1,298 @@
package usecase
import (
"context"
"fmt"
"sync"
"time"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
"gitea.maximumdirect.net/eric/promptkit/internal/validate"
)
type preparedExecutionState uint8
const (
preparedExecutionReady preparedExecutionState = iota
preparedExecutionClaimed
preparedExecutionDiscarded
)
// PreparedExecution owns one frozen, single-use runner execution.
type PreparedExecution struct {
owner *Runner
mu sync.Mutex
state preparedExecutionState
details *domain.PreparedRun
payload *preparedExecutionPayload
}
type preparedExecutionPayload struct {
prepared *domain.PreparedRun
validation validate.PreparedValidation
directKey string
}
// PrepareExecution completes preparation without generation or admission and
// returns a runner-bound, single-use execution.
func (r *Runner) PrepareExecution(ctx context.Context, req domain.RunRequest) (*PreparedExecution, error) {
state, err := r.resolvePreparation(ctx, req, time.Now().UTC())
if err != nil {
return nil, err
}
validationPlan, err := r.prepareValidation(ctx, state.effectiveContract)
if err != nil {
return nil, err
}
structuredOutput, err := r.structuredOutputFromValidationPlan(
state.definition,
state.effectiveContract,
validationPlan,
)
if err != nil {
return nil, err
}
prepared, err := r.completePreparationWithStructuredOutput(ctx, req, state, structuredOutput)
if err != nil {
return nil, err
}
executionSnapshot, err := clonePreparedRun(prepared)
if err != nil {
return nil, fmt.Errorf("%w: failed to copy prepared execution: %v", ErrInvalidRequest, err)
}
details, err := clonePreparedRun(executionSnapshot)
if err != nil {
return nil, fmt.Errorf("%w: failed to copy prepared execution details: %v", ErrInvalidRequest, err)
}
return &PreparedExecution{
owner: r,
state: preparedExecutionReady,
details: details,
payload: &preparedExecutionPayload{
prepared: executionSnapshot,
validation: validationPlan,
directKey: state.effectiveModel.APIKey,
},
}, nil
}
func (r *Runner) prepareValidation(
ctx context.Context,
contract domain.OutputContract,
) (validate.PreparedValidation, error) {
if r.validator == nil {
return noOpPreparedValidation{contract: contract}, nil
}
preparer, ok := r.validator.(validate.ValidationPreparer)
if !ok {
return nil, fmt.Errorf("%w: validator does not support prepared validation", ErrValidation)
}
plan, err := preparer.PrepareValidation(ctx, contract)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
}
if plan == nil {
return nil, fmt.Errorf("%w: validator returned nil prepared validation", ErrValidation)
}
return plan, nil
}
func (r *Runner) structuredOutputFromValidationPlan(
def *domain.PromptDefinition,
contract domain.OutputContract,
plan validate.PreparedValidation,
) (*domain.StructuredOutputSpec, error) {
if contract.ValidationMode != domain.ValidationJSONSchema {
return nil, nil
}
schemaDocument := plan.SchemaDocument()
if schemaDocument == nil {
if r.validator == nil {
return nil, nil
}
return nil, fmt.Errorf("%w: prepared json_schema validation has no schema document", ErrValidation)
}
return structuredOutputSpec(def, schemaDocument), nil
}
// Details returns a fresh credential-redacted copy of the prepared run.
func (p *PreparedExecution) Details() *domain.PreparedRun {
if p == nil {
return nil
}
p.mu.Lock()
detailsSnapshot := p.details
p.mu.Unlock()
details, err := clonePreparedRun(detailsSnapshot)
if err != nil {
return nil
}
return details
}
// Discard invalidates an unclaimed execution and drops its private payload.
func (p *PreparedExecution) Discard() {
if p == nil {
return
}
p.mu.Lock()
if p.state != preparedExecutionReady {
p.mu.Unlock()
return
}
p.state = preparedExecutionDiscarded
payload := p.payload
p.payload = nil
p.mu.Unlock()
payload.clear()
}
// RunPrepared claims and executes one prepared execution owned by this runner.
func (r *Runner) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*domain.RunResult, error) {
payload, err := prepared.claim(r)
if err != nil {
return nil, err
}
defer payload.clear()
runID, err := newRunID()
if err != nil {
return nil, fmt.Errorf("failed to create run id: %w", err)
}
start := time.Now().UTC()
target := payload.prepared.EffectiveModelParams
if err := validateAPIKey(target.APIKeyEnv, payload.directKey, target.APIKeyRequired); err != nil {
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
}
release, err := r.admitRun(ctx, target.BackendID)
if err != nil {
return nil, err
}
defer release()
return r.executePreparedRun(ctx, payload.prepared, payload.directKey, runID, start, func(
ctx context.Context,
artifact *domain.Artifact,
attemptsUsed int,
) (domain.ValidationResult, error) {
result, validationErr := payload.validation.Validate(ctx, artifact)
if validationErr != nil {
return domain.ValidationResult{}, validationErr
}
result.RepairAttempts = attemptsUsed
return result, nil
})
}
func (p *PreparedExecution) claim(owner *Runner) (*preparedExecutionPayload, error) {
if p == nil || owner == nil || p.owner != owner {
return nil, fmt.Errorf("%w: prepared execution does not belong to this runner", ErrInvalidRequest)
}
p.mu.Lock()
defer p.mu.Unlock()
if p.state != preparedExecutionReady || p.payload == nil {
return nil, fmt.Errorf("%w: prepared execution is not ready", ErrInvalidRequest)
}
p.state = preparedExecutionClaimed
payload := p.payload
p.payload = nil
return payload, nil
}
func (p *preparedExecutionPayload) clear() {
if p == nil {
return
}
if p.prepared != nil {
p.prepared.EffectiveModelParams.APIKey = ""
}
p.prepared = nil
p.validation = nil
p.directKey = ""
}
type noOpPreparedValidation struct {
contract domain.OutputContract
}
func (p noOpPreparedValidation) Validate(
ctx context.Context,
_ *domain.Artifact,
) (domain.ValidationResult, error) {
if err := ctx.Err(); err != nil {
return domain.ValidationResult{}, err
}
return domain.ValidationResult{
Status: domain.ValidationSkipped,
Mode: p.contract.ValidationMode,
SchemaPath: p.contract.SchemaPath,
RepairAttempts: p.contract.RepairAttempts,
IsValid: true,
}, nil
}
func (noOpPreparedValidation) SchemaDocument() any {
return nil
}
func clonePreparedRun(source *domain.PreparedRun) (*domain.PreparedRun, error) {
if source == nil {
return nil, nil
}
copied := *source
extraParams, err := jsonvalue.CopyMap(source.EffectiveModelParams.ExtraParams)
if err != nil {
return nil, err
}
copied.EffectiveModelParams.ExtraParams = extraParams
if source.InputHashes != nil {
copied.InputHashes = make(map[string]string, len(source.InputHashes))
for name, hash := range source.InputHashes {
copied.InputHashes[name] = hash
}
}
if source.Messages != nil {
copied.Messages = make([]domain.RenderedMessage, len(source.Messages))
for i, message := range source.Messages {
copied.Messages[i] = message
if message.CacheControl != nil {
cacheControl := *message.CacheControl
copied.Messages[i].CacheControl = &cacheControl
}
}
}
if source.StructuredOutput != nil {
structuredOutput := *source.StructuredOutput
copied.StructuredOutput = &structuredOutput
if source.StructuredOutput.JSONSchema != nil {
jsonSchema := *source.StructuredOutput.JSONSchema
copied.StructuredOutput.JSONSchema = &jsonSchema
schema, copyErr := cloneJSONValue(source.StructuredOutput.JSONSchema.Schema)
if copyErr != nil {
return nil, copyErr
}
copied.StructuredOutput.JSONSchema.Schema = schema
}
}
return &copied, nil
}
func cloneJSONValue(source any) (any, error) {
return jsonvalue.Copy(source)
}

View File

@@ -0,0 +1,510 @@
package usecase
import (
"context"
"errors"
"os"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/validate"
)
type recordingPreparedValidation struct {
contract domain.OutputContract
schemaDocument any
results []domain.ValidationResult
errs []error
artifacts []string
}
func (p *recordingPreparedValidation) Validate(
_ context.Context,
artifact *domain.Artifact,
) (domain.ValidationResult, error) {
p.artifacts = append(p.artifacts, string(artifact.Body))
index := len(p.artifacts) - 1
if index < len(p.errs) && p.errs[index] != nil {
return domain.ValidationResult{}, p.errs[index]
}
if len(p.results) == 0 {
return domain.ValidationResult{
Status: domain.ValidationPassed,
Mode: p.contract.ValidationMode,
IsValid: true,
}, nil
}
if index >= len(p.results) {
index = len(p.results) - 1
}
return p.results[index], nil
}
func (p *recordingPreparedValidation) SchemaDocument() any {
return p.schemaDocument
}
type recordingValidationPreparer struct {
plan *recordingPreparedValidation
prepareErr error
prepareCalls int
directValidateCalls int
}
func (v *recordingValidationPreparer) Validate(
context.Context,
*domain.Artifact,
domain.OutputContract,
) (domain.ValidationResult, error) {
v.directValidateCalls++
return domain.ValidationResult{}, errors.New("live validation must not be used")
}
func (v *recordingValidationPreparer) PrepareValidation(
_ context.Context,
contract domain.OutputContract,
) (validate.PreparedValidation, error) {
v.prepareCalls++
if v.prepareErr != nil {
return nil, v.prepareErr
}
v.plan.contract = contract
return v.plan, nil
}
func TestRunnerPrepareExecutionCompletesWithoutAdmissionOrGeneration(t *testing.T) {
schemaDocument := map[string]any{
"type": "object",
"properties": map[string]any{
"value": map[string]any{"type": "string"},
"": map[string]any{"type": "boolean"},
},
}
def := promptDef(domain.FormatJSON, domain.ValidationJSONSchema, 0)
def.Validation.SchemaPath = "schema.json"
reader := defaultArtifactReader()
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{
SessionID: "prepared-session",
Messages: []domain.RenderedMessage{{
Role: "user",
Content: "original message",
}},
}}
llmClient := &fakeLLM{forbid: true}
validator := &recordingValidationPreparer{
plan: &recordingPreparedValidation{schemaDocument: schemaDocument},
}
admitter := &fakeRunAdmitter{}
profile := defaultExecutionProfile()
profile.ExtraParams = map[string]any{
"metadata": map[string]any{"source": "original"},
}
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
nil,
reader,
renderer,
llmClient,
validator,
admitter,
)
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
APIKey: "direct-test-key",
Inputs: singleInputRef(),
})
if err != nil {
t.Fatalf("prepare execution: %v", err)
}
defer prepared.Discard()
if validator.prepareCalls != 1 || validator.directValidateCalls != 0 {
t.Fatalf(
"validation calls=(prepare=%d direct=%d), want (1, 0)",
validator.prepareCalls,
validator.directValidateCalls,
)
}
if reader.calls != 1 || renderer.calls != 1 {
t.Fatalf("completion calls=(artifact=%d render=%d), want (1, 1)", reader.calls, renderer.calls)
}
if len(admitter.backendIDs) != 0 || llmClient.calls != 0 {
t.Fatalf("prepare invoked execution collaborators: admission=%v generation=%d", admitter.backendIDs, llmClient.calls)
}
first := prepared.Details()
if first == nil {
t.Fatal("prepared details are nil")
}
if first.EffectiveModelParams.APIKey != "" {
t.Fatal("prepared details retained the direct API key")
}
if first.StructuredOutput == nil ||
first.StructuredOutput.JSONSchema == nil ||
!reflect.DeepEqual(first.StructuredOutput.JSONSchema.Schema, schemaDocument) {
t.Fatalf("prepared details have unexpected structured output: %#v", first.StructuredOutput)
}
first.Messages[0].Content = "caller mutation"
first.InputHashes["input"] = "caller mutation"
first.EffectiveModelParams.ExtraParams["metadata"].(map[string]any)["source"] = "caller mutation"
first.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] = "string"
renderer.rendered.Messages[0].Content = "source mutation"
second := prepared.Details()
if second.Messages[0].Content != "original message" ||
second.InputHashes["input"] == "caller mutation" ||
second.EffectiveModelParams.ExtraParams["metadata"].(map[string]any)["source"] != "original" ||
second.StructuredOutput.JSONSchema.Schema.(map[string]any)["type"] != "object" {
t.Fatalf("details did not preserve an independent snapshot: %#v", second)
}
}
func TestRunnerRunPreparedRechecksEnvironmentCredentialBeforeAdmission(t *testing.T) {
const environmentName = "PROMPTKIT_PREPARED_EXECUTION_TEST_KEY"
t.Setenv(environmentName, "available-during-preparation")
profile := defaultExecutionProfile()
profile.APIKeyEnv = environmentName
validator := &recordingValidationPreparer{plan: &recordingPreparedValidation{}}
admitter := &fakeRunAdmitter{}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "unexpected"}}
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
nil,
defaultArtifactReader(),
defaultRenderer(),
llmClient,
validator,
admitter,
)
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
})
if err != nil {
t.Fatalf("prepare execution: %v", err)
}
if err := os.Unsetenv(environmentName); err != nil {
t.Fatalf("unset credential environment: %v", err)
}
result, err := runner.RunPrepared(context.Background(), prepared)
if result != nil {
t.Fatalf("credential failure returned partial result: %+v", result)
}
if !errors.Is(err, ErrInvalidRequest) || !errors.Is(err, ErrAPIKeyEnvMissing) {
t.Fatalf("credential error identities are missing: %v", err)
}
if len(admitter.backendIDs) != 0 || llmClient.calls != 0 {
t.Fatalf("credential failure reached admission or generation: admission=%v generation=%d", admitter.backendIDs, llmClient.calls)
}
if _, err := runner.RunPrepared(context.Background(), prepared); !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("credential failure did not consume execution: %v", err)
}
}
func TestRunnerRunPreparedKeepsDirectCredentialOutOfMetadata(t *testing.T) {
const directKey = "direct-prepared-test-key"
profile := defaultExecutionProfile()
profile.APIKeyRequired = true
client := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
nil,
defaultArtifactReader(),
defaultRenderer(),
client,
&recordingValidationPreparer{plan: &recordingPreparedValidation{}},
nil,
)
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
APIKey: directKey,
Inputs: singleInputRef(),
})
if err != nil {
t.Fatalf("prepare execution: %v", err)
}
if details := prepared.Details(); details.EffectiveModelParams.APIKey != "" {
t.Fatal("prepared details retained direct credential")
}
result, err := runner.RunPrepared(context.Background(), prepared)
if err != nil {
t.Fatalf("run prepared: %v", err)
}
if client.lastReq.Target.APIKey != directKey {
t.Fatal("generation did not receive direct credential")
}
if result.EffectiveModelParams.APIKey != "" {
t.Fatal("run result retained direct credential")
}
}
func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *testing.T) {
validator := &recordingValidationPreparer{
plan: &recordingPreparedValidation{
results: []domain.ValidationResult{
{
Status: domain.ValidationFailed,
Mode: domain.ValidationJSON,
Errors: []string{"invalid"},
IsValid: false,
},
{
Status: domain.ValidationPassed,
Mode: domain.ValidationJSON,
IsValid: true,
},
},
},
}
repairer := &fakeRepairer{
responses: []*domain.GenerateResponse{{Content: `{"repaired":true}`}},
}
admitter := &fakeRunAdmitter{}
reader := defaultArtifactReader()
renderer := defaultRenderer()
runner := NewRunnerWithRepairer(
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
reader,
renderer,
&fakeLLM{resp: &domain.GenerateResponse{Content: `{"broken":true}`}},
validator,
repairer,
admitter,
)
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
})
if err != nil {
t.Fatalf("prepare execution: %v", err)
}
result, err := runner.RunPrepared(context.Background(), prepared)
if err != nil {
t.Fatalf("run prepared: %v", err)
}
if !reflect.DeepEqual(validator.plan.artifacts, []string{`{"broken":true}`, `{"repaired":true}`}) {
t.Fatalf("prepared validation artifacts=%#v", validator.plan.artifacts)
}
if validator.directValidateCalls != 0 || repairer.calls != 1 {
t.Fatalf("validation/repair calls=(direct=%d repair=%d), want (0, 1)", validator.directValidateCalls, repairer.calls)
}
if result.Validation.Status != domain.ValidationPassed || result.Validation.RepairAttempts != 1 {
t.Fatalf("unexpected repaired validation result: %+v", result.Validation)
}
if admitter.releaseCalls != 1 {
t.Fatalf("admission releases=%d, want 1", admitter.releaseCalls)
}
if reader.calls != 1 || renderer.calls != 1 {
t.Fatalf("execution reopened preparation sources: artifact=%d render=%d", reader.calls, renderer.calls)
}
}
func TestRunnerRunPreparedReleasesAdmissionAcrossExecutionErrors(t *testing.T) {
generationFailure := errors.New("generation failed")
validationFailure := errors.New("validation failed")
repairFailure := errors.New("repair failed")
tests := []struct {
name string
generationErr error
validation *recordingPreparedValidation
repairer *fakeRepairer
wantError error
}{
{
name: "generation failure",
generationErr: generationFailure,
validation: &recordingPreparedValidation{},
wantError: ErrLLMGenerate,
},
{
name: "validation failure",
validation: &recordingPreparedValidation{
errs: []error{validationFailure},
},
wantError: ErrValidation,
},
{
name: "repair failure",
validation: &recordingPreparedValidation{
results: []domain.ValidationResult{{
Status: domain.ValidationFailed,
Mode: domain.ValidationJSON,
Errors: []string{"invalid"},
IsValid: false,
}},
},
repairer: &fakeRepairer{err: repairFailure},
wantError: ErrValidation,
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
def := promptDef(domain.FormatJSON, domain.ValidationJSON, 1)
if test.repairer == nil {
def.Validation.RepairAttempts = 0
}
validator := &recordingValidationPreparer{plan: test.validation}
admitter := &fakeRunAdmitter{}
runner := NewRunnerWithRepairer(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{
resp: &domain.GenerateResponse{Content: `{"value":true}`},
err: test.generationErr,
},
validator,
test.repairer,
admitter,
)
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
})
if err != nil {
t.Fatalf("prepare execution: %v", err)
}
result, err := runner.RunPrepared(context.Background(), prepared)
if result != nil || !errors.Is(err, test.wantError) {
t.Fatalf("run prepared=(%+v, %v), want %v", result, err, test.wantError)
}
if len(admitter.backendIDs) != 1 || admitter.releaseCalls != 1 {
t.Fatalf(
"admission calls=%#v releases=%d, want one each",
admitter.backendIDs,
admitter.releaseCalls,
)
}
})
}
}
func TestRunnerPreparedExecutionOwnershipUseAndDiscard(t *testing.T) {
newRunner := func() *Runner {
return NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
&recordingValidationPreparer{plan: &recordingPreparedValidation{}},
nil,
)
}
request := domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
}
owner := newRunner()
prepared, err := owner.PrepareExecution(context.Background(), request)
if err != nil {
t.Fatalf("prepare execution: %v", err)
}
if _, err := newRunner().RunPrepared(context.Background(), prepared); !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("foreign runner error=%v, want ErrInvalidRequest", err)
}
if _, err := owner.RunPrepared(context.Background(), prepared); err != nil {
t.Fatalf("owner run prepared: %v", err)
}
if _, err := owner.RunPrepared(context.Background(), prepared); !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("second owner run error=%v, want ErrInvalidRequest", err)
}
if prepared.Details() == nil {
t.Fatal("details unavailable after execution")
}
discarded, err := owner.PrepareExecution(context.Background(), request)
if err != nil {
t.Fatalf("prepare discarded execution: %v", err)
}
discarded.Discard()
discarded.Discard()
if _, err := owner.RunPrepared(context.Background(), discarded); !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("discarded execution error=%v, want ErrInvalidRequest", err)
}
if discarded.Details() == nil {
t.Fatal("details unavailable after discard")
}
}
func TestRunnerPreparedExecutionWithoutValidatorSkipsValidation(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{resp: &domain.GenerateResponse{Content: `{}`}},
nil,
nil,
)
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
})
if err != nil {
t.Fatalf("prepare execution: %v", err)
}
result, err := runner.RunPrepared(context.Background(), prepared)
if err != nil {
t.Fatalf("run prepared: %v", err)
}
if result.Validation.Status != domain.ValidationSkipped || !result.Validation.IsValid {
t.Fatalf("unexpected no-validator result: %+v", result.Validation)
}
}
func TestRunnerPrepareExecutionRequiresValidationPreparer(t *testing.T) {
reader := defaultArtifactReader()
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
reader,
defaultRenderer(),
&fakeLLM{forbid: true},
&fakeValidator{},
nil,
)
prepared, err := runner.PrepareExecution(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
})
if prepared != nil || !errors.Is(err, ErrValidation) {
t.Fatalf("prepare execution=(%+v, %v), want ErrValidation", prepared, err)
}
if reader.calls != 0 {
t.Fatalf("unsupported validator allowed completion, artifact calls=%d", reader.calls)
}
}

View File

@@ -0,0 +1,103 @@
package usecase
import (
"context"
"errors"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
type resolvedProfileSelection struct {
id string
profile *domain.ExecutionProfile
backend *domain.Backend
}
func (r *Runner) resolveProfileSelection(
ctx context.Context,
profileID string,
) (*resolvedProfileSelection, error) {
normalizedID := strings.TrimSpace(profileID)
if normalizedID == "" {
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
}
if r == nil || r.profiles == nil {
return nil, fmt.Errorf("%w: profile repository is not configured", ErrProfileLoad)
}
selectedProfile, err := r.profiles.GetProfile(ctx, normalizedID)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
}
if selectedProfile == nil {
return nil, fmt.Errorf("%w: profile repository returned nil profile", ErrProfileLoad)
}
profileValue := *selectedProfile
profileValue.BackendID = strings.TrimSpace(profileValue.BackendID)
var selectedBackend *domain.Backend
if profileValue.BackendID != "" {
if r.backends == nil {
return nil, fmt.Errorf("%w: backend %q cannot be resolved", ErrProfileLoad, profileValue.BackendID)
}
backendValue, err := r.backends.GetBackend(profileValue.BackendID)
if err != nil {
return nil, fmt.Errorf("%w: backend %q: %w", ErrProfileLoad, profileValue.BackendID, err)
}
selectedBackend = &backendValue
}
return &resolvedProfileSelection{
id: normalizedID,
profile: &profileValue,
backend: selectedBackend,
}, nil
}
func validateResolvedExecutionTarget(target domain.ExecutionTarget) error {
if strings.TrimSpace(target.Endpoint) == "" {
return errors.New("execution endpoint is required")
}
if strings.TrimSpace(target.Model) == "" {
return errors.New("execution model is required")
}
return nil
}
// InspectProfile resolves one explicit profile without prompt or execution work.
func (r *Runner) InspectProfile(
ctx context.Context,
profileID string,
) (*domain.ProfileInspection, error) {
normalizedID := strings.TrimSpace(profileID)
if normalizedID == "" {
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
}
select {
case <-ctx.Done():
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, ctx.Err())
default:
}
selection, err := r.resolveProfileSelection(ctx, normalizedID)
if err != nil {
return nil, err
}
target, _, err := resolveExecutionTarget(selection.backend, selection.profile, nil)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
}
if err := validateResolvedExecutionTarget(target); err != nil {
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
}
target.APIKey = ""
return &domain.ProfileInspection{
ProfileID: selection.id,
EffectiveModelParams: target,
APIKeyRequired: target.APIKeyRequired,
}, nil
}

View File

@@ -0,0 +1,213 @@
package usecase
import (
"context"
"errors"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
)
type inspectionProfileRepository struct {
profile *domain.ExecutionProfile
err error
calls int
id string
}
func (r *inspectionProfileRepository) GetProfile(
_ context.Context,
id string,
) (*domain.ExecutionProfile, error) {
r.calls++
r.id = id
if r.err != nil {
return nil, r.err
}
return r.profile, nil
}
type inspectionBackendResolver struct {
backend domain.Backend
err error
calls int
id string
}
func (r *inspectionBackendResolver) GetBackend(id string) (domain.Backend, error) {
r.calls++
r.id = id
if r.err != nil {
return domain.Backend{}, r.err
}
return r.backend, nil
}
func TestRunnerInspectProfileResolvesProfileAndBackendOnce(t *testing.T) {
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
ID: "profile",
BackendID: " backend ",
Model: "profile-model",
Temperature: 0.4,
MaxTokens: 32,
TimeoutSeconds: 45,
ServiceTier: "priority",
ReasoningEffort: "high",
ExtraParams: map[string]any{
"profile": "value",
},
}}
backends := &inspectionBackendResolver{backend: domain.Backend{
ID: "backend",
Endpoint: "https://backend.example/v1",
APIKeyEnv: "BACKEND_KEY",
ExtraParams: map[string]any{
"backend": "value",
},
}}
runner := &Runner{profiles: profiles, backends: backends}
inspection, err := runner.InspectProfile(context.Background(), " profile ")
if err != nil {
t.Fatalf("inspect profile: %v", err)
}
if profiles.calls != 1 || profiles.id != "profile" {
t.Fatalf("profile lookup=(calls=%d id=%q), want one exact lookup", profiles.calls, profiles.id)
}
if backends.calls != 1 || backends.id != "backend" {
t.Fatalf("backend lookup=(calls=%d id=%q), want one exact lookup", backends.calls, backends.id)
}
if profiles.profile.BackendID != " backend " {
t.Fatalf("inspection mutated repository profile backend: %q", profiles.profile.BackendID)
}
wantTarget := domain.ExecutionTarget{
BackendID: "backend",
Endpoint: "https://backend.example/v1",
Model: "profile-model",
Temperature: 0.4,
MaxTokens: 32,
TopP: defaults.ExecutionTargetDefault().TopP,
TimeoutSeconds: 45,
ServiceTier: "priority",
ReasoningEffort: "high",
APIKeyEnv: "BACKEND_KEY",
ExtraParams: map[string]any{
"profile": "value",
},
}
if inspection.ProfileID != "profile" || inspection.APIKeyRequired ||
!reflect.DeepEqual(inspection.EffectiveModelParams, wantTarget) {
t.Fatalf("inspection=%#v, want profile=%q target=%#v", inspection, "profile", wantTarget)
}
}
func TestRunnerInspectProfileDoesNotNeedExecutionCollaboratorsOrCredentials(t *testing.T) {
t.Setenv("PROMPTKIT_INSPECTION_TEST_KEY", "")
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
ID: "endpoint-only",
Endpoint: "https://profile.example/v1",
Model: "profile-model",
APIKeyEnv: "PROMPTKIT_INSPECTION_TEST_KEY",
}}
runner := &Runner{profiles: profiles}
inspection, err := runner.InspectProfile(context.Background(), "endpoint-only")
if err != nil {
t.Fatalf("inspect endpoint-only profile: %v", err)
}
if inspection.EffectiveModelParams.BackendID != "" ||
inspection.EffectiveModelParams.APIKeyEnv != "PROMPTKIT_INSPECTION_TEST_KEY" ||
inspection.APIKeyRequired {
t.Fatalf("unexpected endpoint-only inspection: %#v", inspection)
}
}
func TestRunnerInspectProfileDirectCredentialRequirementClearsBackendEnvironment(t *testing.T) {
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
ID: "direct-key",
BackendID: "backend",
Model: "profile-model",
APIKeyRequired: true,
}}
backends := &inspectionBackendResolver{backend: domain.Backend{
ID: "backend",
Endpoint: "https://backend.example/v1",
APIKeyEnv: "BACKEND_KEY",
}}
inspection, err := (&Runner{profiles: profiles, backends: backends}).InspectProfile(
context.Background(),
"direct-key",
)
if err != nil {
t.Fatalf("inspect direct-key profile: %v", err)
}
if !inspection.APIKeyRequired || inspection.EffectiveModelParams.APIKeyEnv != "" {
t.Fatalf("credential requirement was not resolved exclusively: %#v", inspection)
}
}
func TestRunnerInspectProfileClassifiesFailuresWithoutRepositoryWorkAfterCancellation(t *testing.T) {
t.Run("blank ID", func(t *testing.T) {
profiles := &inspectionProfileRepository{}
_, err := (&Runner{profiles: profiles}).InspectProfile(context.Background(), " \t ")
if !errors.Is(err, ErrInvalidRequest) || profiles.calls != 0 {
t.Fatalf("blank inspection=(%v, calls=%d), want invalid request without lookup", err, profiles.calls)
}
})
t.Run("canceled context", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
profiles := &inspectionProfileRepository{}
_, err := (&Runner{profiles: profiles}).InspectProfile(ctx, "profile")
if !errors.Is(err, ErrProfileLoad) || !errors.Is(err, context.Canceled) || profiles.calls != 0 {
t.Fatalf("canceled inspection=(%v, calls=%d), want profile load and context identities without lookup", err, profiles.calls)
}
})
t.Run("missing profile", func(t *testing.T) {
profiles := &inspectionProfileRepository{err: profile.ErrProfileNotFound}
_, err := (&Runner{profiles: profiles}).InspectProfile(context.Background(), "missing")
if !errors.Is(err, ErrProfileLoad) || !errors.Is(err, profile.ErrProfileNotFound) {
t.Fatalf("missing profile error=%v, want profile load and not-found identities", err)
}
})
t.Run("unknown backend", func(t *testing.T) {
backendErr := errors.New("unknown backend")
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
ID: "profile", BackendID: "backend", Model: "profile-model",
}}
backends := &inspectionBackendResolver{err: backendErr}
_, err := (&Runner{profiles: profiles, backends: backends}).InspectProfile(context.Background(), "profile")
if !errors.Is(err, ErrProfileLoad) || !errors.Is(err, backendErr) {
t.Fatalf("unknown backend error=%v, want profile load and backend identities", err)
}
})
t.Run("defensive invalid dependencies", func(t *testing.T) {
cases := []struct {
name string
runner *Runner
}{
{name: "nil repository", runner: &Runner{}},
{name: "nil profile", runner: &Runner{profiles: &inspectionProfileRepository{}}},
{name: "invalid target", runner: &Runner{profiles: &inspectionProfileRepository{
profile: &domain.ExecutionProfile{ID: "profile", Endpoint: "https://profile.example/v1"},
}}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
_, err := tc.runner.InspectProfile(context.Background(), "profile")
if !errors.Is(err, ErrProfileLoad) {
t.Fatalf("inspection error=%v, want profile load", err)
}
})
}
})
}

View File

@@ -0,0 +1,73 @@
package usecase
import (
"context"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
type resolvedPromptDefinition struct {
definition *domain.PromptDefinition
hash string
}
func (r *Runner) resolvePromptDefinition(
ctx context.Context,
promptID string,
promptVersion string,
) (*resolvedPromptDefinition, error) {
if strings.TrimSpace(promptID) == "" {
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
}
if r == nil || r.promptDefs == nil {
return nil, fmt.Errorf("%w: prompt repository is not configured", ErrPromptLoad)
}
definition, err := r.promptDefs.GetPromptDefinition(ctx, promptID, promptVersion)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
}
if definition == nil {
return nil, fmt.Errorf("%w: prompt repository returned nil definition", ErrPromptLoad)
}
hash, err := hashPromptDefinition(definition)
if err != nil {
return nil, fmt.Errorf("%w: failed to hash prompt definition: %v", ErrPromptLoad, err)
}
return &resolvedPromptDefinition{definition: definition, hash: hash}, nil
}
// InspectPrompt resolves one explicit prompt without execution work.
func (r *Runner) InspectPrompt(
ctx context.Context,
promptID string,
promptVersion string,
) (*domain.PromptInspection, error) {
if strings.TrimSpace(promptID) == "" {
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
}
select {
case <-ctx.Done():
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, ctx.Err())
default:
}
selection, err := r.resolvePromptDefinition(ctx, promptID, promptVersion)
if err != nil {
return nil, err
}
inputs := make([]domain.PromptInput, len(selection.definition.Inputs))
copy(inputs, selection.definition.Inputs)
return &domain.PromptInspection{
PromptID: selection.definition.ID,
PromptVersion: selection.definition.Version,
PromptHash: selection.hash,
DefaultProfileID: selection.definition.DefaultProfile,
Inputs: inputs,
OutputContract: selection.definition.Validation,
}, nil
}

View File

@@ -0,0 +1,157 @@
package usecase
import (
"context"
"errors"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
)
type inspectionPromptRepository struct {
definition *domain.PromptDefinition
err error
calls int
id string
version string
}
func (r *inspectionPromptRepository) GetPromptDefinition(
_ context.Context,
id string,
version string,
) (*domain.PromptDefinition, error) {
r.calls++
r.id = id
r.version = version
if r.err != nil {
return nil, r.err
}
return r.definition, nil
}
func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t *testing.T) {
definition := &domain.PromptDefinition{
ID: "normalized.prompt",
Version: "1.2.3",
DefaultProfile: "not-resolved",
Inputs: []domain.PromptInput{
{Name: "document", Required: true, ContentType: "text/plain", Description: "Source document."},
{Name: "audience", ContentType: "text/plain", Description: "Intended reader."},
},
Validation: domain.OutputContract{
Format: domain.FormatJSON,
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "schemas/result.json",
RepairAttempts: 2,
},
}
repository := &inspectionPromptRepository{definition: definition}
runner := &Runner{promptDefs: repository}
inspection, err := runner.InspectPrompt(context.Background(), " prompt-id ", " version ")
if err != nil {
t.Fatalf("inspect prompt: %v", err)
}
wantHash, err := hashPromptDefinition(definition)
if err != nil {
t.Fatalf("hash prompt definition: %v", err)
}
if repository.calls != 1 || repository.id != " prompt-id " || repository.version != " version " {
t.Fatalf("prompt lookup=(calls=%d id=%q version=%q), want one unchanged lookup", repository.calls, repository.id, repository.version)
}
if inspection.PromptID != definition.ID ||
inspection.PromptVersion != definition.Version ||
inspection.PromptHash != wantHash ||
inspection.DefaultProfileID != definition.DefaultProfile ||
!reflect.DeepEqual(inspection.Inputs, definition.Inputs) ||
inspection.OutputContract != definition.Validation {
t.Fatalf("inspection=%#v, want definition metadata", inspection)
}
inspection.Inputs[0].Name = "changed"
second, err := runner.InspectPrompt(context.Background(), " prompt-id ", " version ")
if err != nil {
t.Fatalf("inspect prompt again: %v", err)
}
if definition.Inputs[0].Name != "document" || second.Inputs[0].Name != "document" {
t.Fatalf("inspection input mutation escaped caller result: definition=%#v next=%#v", definition.Inputs, second.Inputs)
}
}
func TestRunnerInspectPromptClassifiesFailuresWithoutRepositoryWorkAfterCancellation(t *testing.T) {
t.Run("blank ID", func(t *testing.T) {
repository := &inspectionPromptRepository{}
_, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), " \t ", "version")
if !errors.Is(err, ErrInvalidRequest) || repository.calls != 0 {
t.Fatalf("blank inspection=(%v, calls=%d), want invalid request without lookup", err, repository.calls)
}
})
t.Run("canceled context", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
repository := &inspectionPromptRepository{}
_, err := (&Runner{promptDefs: repository}).InspectPrompt(ctx, "prompt", "version")
if !errors.Is(err, ErrPromptLoad) || !errors.Is(err, context.Canceled) || repository.calls != 0 {
t.Fatalf("canceled inspection=(%v, calls=%d), want prompt load and context identities without lookup", err, repository.calls)
}
})
t.Run("missing prompt", func(t *testing.T) {
repository := &inspectionPromptRepository{err: promptdef.ErrPromptDefinitionNotFound}
_, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), "missing", "version")
if !errors.Is(err, ErrPromptLoad) || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
t.Fatalf("missing prompt error=%v, want prompt load and not-found identities", err)
}
})
t.Run("defensive prompt dependencies", func(t *testing.T) {
var nilRunner *Runner
cases := []struct {
name string
runner *Runner
}{
{name: "nil runner", runner: nilRunner},
{name: "nil repository", runner: &Runner{}},
{name: "nil definition", runner: &Runner{promptDefs: &inspectionPromptRepository{}}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
_, err := tc.runner.InspectPrompt(context.Background(), "prompt", "version")
if !errors.Is(err, ErrPromptLoad) {
t.Fatalf("inspection error=%v, want prompt load", err)
}
})
}
})
}
func TestRunnerPrepareUsesThePromptInspectionSelectionAndHash(t *testing.T) {
definition := promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)
repository := &fakePromptRepo{def: definition}
runner := NewRunner(
repository,
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
&fakeArtifactReader{},
&fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hello"}}}},
nil,
nil,
nil,
)
inspection, err := runner.InspectPrompt(context.Background(), definition.ID, definition.Version)
if err != nil {
t.Fatalf("inspect prompt: %v", err)
}
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: definition.ID, PromptVersion: definition.Version, ProfileID: "exec"})
if err != nil {
t.Fatalf("prepare prompt: %v", err)
}
if inspection.PromptHash != prepared.PromptHash {
t.Fatalf("inspection hash=%q, preparation hash=%q", inspection.PromptHash, prepared.PromptHash)
}
}

View File

@@ -131,29 +131,46 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
return nil, err
}
if r.admitter != nil {
release, admitErr := r.admitter.Admit(ctx, state.effectiveModel.BackendID)
if admitErr != nil {
if errors.Is(admitErr, capacity.ErrCapacityExceeded) {
return nil, fmt.Errorf(
"backend %q admission: %w",
state.effectiveModel.BackendID,
admitErr,
)
}
return nil, admitErr
}
defer release()
release, err := r.admitRun(ctx, state.effectiveModel.BackendID)
if err != nil {
return nil, err
}
defer release()
prepared, err := r.completePreparation(ctx, req, state)
if err != nil {
return nil, err
}
directAPIKey := state.effectiveModel.APIKey
return r.executePreparedRun(ctx, prepared, directAPIKey, runID, start, func(
ctx context.Context,
artifact *domain.Artifact,
attemptsUsed int,
) (domain.ValidationResult, error) {
return r.validateOutput(ctx, artifact, prepared.OutputContract, attemptsUsed)
})
}
type preparedValidationFunc func(
context.Context,
*domain.Artifact,
int,
) (domain.ValidationResult, error)
func (r *Runner) executePreparedRun(
ctx context.Context,
prepared *domain.PreparedRun,
directAPIKey string,
runID string,
start time.Time,
validateArtifact preparedValidationFunc,
) (*domain.RunResult, error) {
executionTarget := prepared.EffectiveModelParams
executionTarget.APIKey = directAPIKey
genResp, err := r.llm.Generate(ctx, domain.GenerateRequest{
Prompt: domain.RenderedPrompt{SessionID: prepared.SessionID, Messages: prepared.Messages},
Target: prepared.EffectiveModelParams,
Target: executionTarget,
TargetPresence: prepared.TargetPresence,
StructuredOutput: prepared.StructuredOutput,
})
@@ -165,7 +182,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
}
outputArtifact := buildOutputArtifact(genResp.Content, prepared.OutputContract.Format)
validationResult, err := r.validateOutput(ctx, &outputArtifact, prepared.OutputContract, 0)
validationResult, err := validateArtifact(ctx, &outputArtifact, 0)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
}
@@ -179,7 +196,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
PreviousOutput: genResp.Content,
ValidationErrors: validationResult.Errors,
SessionID: prepared.SessionID,
Target: prepared.EffectiveModelParams,
Target: executionTarget,
StructuredOutput: prepared.StructuredOutput,
Attempt: attemptsUsed,
MaxAttempts: prepared.OutputContract.RepairAttempts,
@@ -195,7 +212,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
genResp = repairResp
outputArtifact = buildOutputArtifact(genResp.Content, prepared.OutputContract.Format)
validationResult, err = r.validateOutput(ctx, &outputArtifact, prepared.OutputContract, attemptsUsed)
validationResult, err = validateArtifact(ctx, &outputArtifact, attemptsUsed)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrValidation, err)
}
@@ -203,6 +220,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
}
end := time.Now().UTC()
executionTarget.APIKey = ""
return &domain.RunResult{
RunID: runID,
@@ -218,7 +236,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
SelectedBackendID: prepared.SelectedBackendID,
ModelName: prepared.EffectiveModelParams.Model,
Endpoint: prepared.EffectiveModelParams.Endpoint,
EffectiveModelParams: prepared.EffectiveModelParams,
EffectiveModelParams: executionTarget,
InputHashes: prepared.InputHashes,
Usage: genResp.Usage,
StartTime: start,
@@ -248,14 +266,12 @@ func (r *Runner) resolvePreparation(
return nil, fmt.Errorf("%w: session_id: %v", ErrInvalidRequest, err)
}
def, err := r.promptDefs.GetPromptDefinition(ctx, req.PromptID, req.PromptVersion)
promptSelection, err := r.resolvePromptDefinition(ctx, req.PromptID, req.PromptVersion)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
}
promptDefinitionHash, err := hashPromptDefinition(def)
if err != nil {
return nil, fmt.Errorf("%w: failed to hash prompt definition: %v", ErrPromptLoad, err)
return nil, err
}
def := promptSelection.definition
promptDefinitionHash := promptSelection.hash
selectedProfileID := strings.TrimSpace(req.ProfileID)
if selectedProfileID == "" {
@@ -265,34 +281,18 @@ func (r *Runner) resolvePreparation(
return nil, fmt.Errorf("%w: %w: profile id is required either in request or prompt default_profile", ErrInvalidRequest, ErrProfileRequired)
}
execProfile, err := r.profiles.GetProfile(ctx, selectedProfileID)
selection, err := r.resolveProfileSelection(ctx, selectedProfileID)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
return nil, err
}
var selectedBackend *domain.Backend
if backendID := strings.TrimSpace(execProfile.BackendID); backendID != "" {
execProfile.BackendID = backendID
if r.backends == nil {
return nil, fmt.Errorf("%w: backend %q cannot be resolved", ErrProfileLoad, backendID)
}
resolvedBackend, resolveErr := r.backends.GetBackend(backendID)
if resolveErr != nil {
return nil, fmt.Errorf("%w: backend %q: %w", ErrProfileLoad, backendID, resolveErr)
}
selectedBackend = &resolvedBackend
}
effectiveModel, targetPresence, err := resolveExecutionTarget(selectedBackend, execProfile, req.Execution)
effectiveModel, targetPresence, err := resolveExecutionTarget(selection.backend, selection.profile, req.Execution)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
}
effectiveModel.APIKey = req.APIKey
if strings.TrimSpace(effectiveModel.Endpoint) == "" {
return nil, fmt.Errorf("%w: execution endpoint is required", ErrInvalidRequest)
}
if strings.TrimSpace(effectiveModel.Model) == "" {
return nil, fmt.Errorf("%w: execution model is required", ErrInvalidRequest)
if err := validateResolvedExecutionTarget(effectiveModel); err != nil {
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
}
if err := validateAPIKey(effectiveModel.APIKeyEnv, effectiveModel.APIKey, effectiveModel.APIKeyRequired); err != nil {
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
@@ -303,7 +303,7 @@ func (r *Runner) resolvePreparation(
definition: def,
directSessionID: directSessionID,
promptDefinitionHash: promptDefinitionHash,
selectedProfileID: selectedProfileID,
selectedProfileID: selection.id,
effectiveModel: effectiveModel,
targetPresence: targetPresence,
effectiveContract: effectiveContract,
@@ -324,7 +324,15 @@ func (r *Runner) completePreparation(
if err != nil {
return nil, err
}
return r.completePreparationWithStructuredOutput(ctx, req, state, structuredOutput)
}
func (r *Runner) completePreparationWithStructuredOutput(
ctx context.Context,
req domain.RunRequest,
state *preparationState,
structuredOutput *domain.StructuredOutputSpec,
) (*domain.PreparedRun, error) {
resolvedInputs := make(map[string]*domain.Artifact, len(req.Inputs))
inputHashes := make(map[string]string, len(req.Inputs))
for name, ref := range req.Inputs {
@@ -354,13 +362,15 @@ func (r *Runner) completePreparation(
}
end := time.Now().UTC()
effectiveModel := state.effectiveModel
effectiveModel.APIKey = ""
return &domain.PreparedRun{
PromptID: state.definition.ID,
PromptVersion: state.definition.Version,
PromptHash: state.promptDefinitionHash,
SelectedProfileID: state.selectedProfileID,
SelectedBackendID: state.effectiveModel.BackendID,
EffectiveModelParams: state.effectiveModel,
EffectiveModelParams: effectiveModel,
TargetPresence: state.targetPresence,
OutputContract: state.effectiveContract,
StructuredOutput: structuredOutput,
@@ -374,6 +384,20 @@ func (r *Runner) completePreparation(
}, nil
}
func (r *Runner) admitRun(ctx context.Context, backendID string) (func(), error) {
if r.admitter == nil {
return func() {}, nil
}
release, err := r.admitter.Admit(ctx, backendID)
if err != nil {
if errors.Is(err, capacity.ErrCapacityExceeded) && strings.TrimSpace(backendID) != "" {
return nil, &CapacityError{BackendID: backendID}
}
return nil, err
}
return release, nil
}
func (r *Runner) resolveStructuredOutput(ctx context.Context, def *domain.PromptDefinition, contract domain.OutputContract) (*domain.StructuredOutputSpec, error) {
if contract.ValidationMode != domain.ValidationJSONSchema {
return nil, nil
@@ -389,14 +413,18 @@ func (r *Runner) resolveStructuredOutput(ctx context.Context, def *domain.Prompt
return nil, fmt.Errorf("%w: failed to load json schema for structured output: %v", ErrValidation, err)
}
return structuredOutputSpec(def, schemaDoc), nil
}
func structuredOutputSpec(def *domain.PromptDefinition, schemaDocument any) *domain.StructuredOutputSpec {
return &domain.StructuredOutputSpec{
Type: domain.StructuredOutputJSONSchema,
JSONSchema: &domain.StructuredOutputJSONSpec{
Name: deriveStructuredSchemaName(def.ID, def.Version),
Strict: true,
Schema: schemaDoc,
Schema: schemaDocument,
},
}, nil
}
}
func deriveStructuredSchemaName(promptID string, promptVersion string) string {

View File

@@ -1357,14 +1357,14 @@ func TestRunnerAdmissionUsesResolvedBackendIdentity(t *testing.T) {
func TestRunnerAdmissionFailureSkipsCompletionCollaborators(t *testing.T) {
tests := []struct {
name string
admissionError error
wantBackendContext bool
name string
admissionError error
wantCapacityType bool
}{
{
name: "capacity exhausted",
admissionError: capacity.ErrCapacityExceeded,
wantBackendContext: true,
name: "capacity exhausted",
admissionError: capacity.ErrCapacityExceeded,
wantCapacityType: true,
},
{
name: "context canceled",
@@ -1414,8 +1414,16 @@ func TestRunnerAdmissionFailureSkipsCompletionCollaborators(t *testing.T) {
if errors.Is(err, ErrInvalidRequest) || errors.Is(err, ErrLLMGenerate) {
t.Fatalf("admission error was recategorized: %v", err)
}
if tc.wantBackendContext && !strings.Contains(err.Error(), "custom") {
t.Fatalf("capacity error lacks backend context: %v", err)
var capacityErr *CapacityError
if tc.wantCapacityType {
if !errors.As(err, &capacityErr) {
t.Fatalf("capacity error=%v, want internal typed identity", err)
}
if capacityErr.BackendID != "custom" {
t.Fatalf("capacity backend ID=%q, want custom", capacityErr.BackendID)
}
} else if errors.As(err, &capacityErr) {
t.Fatalf("non-capacity admission error exposed typed capacity identity: %v", err)
}
if !reflect.DeepEqual(admitter.backendIDs, []string{"custom"}) {
t.Fatalf("admitted backend IDs=%#v, want custom", admitter.backendIDs)
@@ -1682,36 +1690,6 @@ func TestRunnerRunSelectedProfileBeatsBuiltInDefault(t *testing.T) {
}
}
func TestRunnerRunBuiltInDefaultsUsedWhenProfileOmitsOptionalFields(t *testing.T) {
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"},
}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if res.EffectiveModelParams.Temperature != defaults.ExecutionDefaultTemperature {
t.Fatalf("expected default temperature %v, got %v", defaults.ExecutionDefaultTemperature, res.EffectiveModelParams.Temperature)
}
if res.EffectiveModelParams.TopP != defaults.ExecutionDefaultTopP {
t.Fatalf("expected default top_p %v, got %v", defaults.ExecutionDefaultTopP, res.EffectiveModelParams.TopP)
}
if res.EffectiveModelParams.MaxTokens != defaults.ExecutionDefaultMaxTokens {
t.Fatalf("expected default max_tokens %d, got %d", defaults.ExecutionDefaultMaxTokens, res.EffectiveModelParams.MaxTokens)
}
if res.EffectiveModelParams.TimeoutSeconds != defaults.ExecutionDefaultTimeoutSeconds {
t.Fatalf("expected default timeout_seconds %d, got %d", defaults.ExecutionDefaultTimeoutSeconds, res.EffectiveModelParams.TimeoutSeconds)
}
}
func TestRunnerRunAPIKeyEnvResolvesFromEnvironment(t *testing.T) {
t.Setenv("PROMPTKIT_TEST_API_KEY", "secret")
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
@@ -1757,7 +1735,7 @@ func TestRunnerRunDirectAPIKeyBypassesMissingEnvAndReachesLLM(t *testing.T) {
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{
result, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
APIKey: directKey,
@@ -1769,6 +1747,9 @@ func TestRunnerRunDirectAPIKeyBypassesMissingEnvAndReachesLLM(t *testing.T) {
if llmClient.lastReq.Target.APIKey != directKey {
t.Fatalf("expected direct API key to reach LLM request")
}
if result.EffectiveModelParams.APIKey != "" {
t.Fatal("run result retained direct API key")
}
if llmClient.lastReq.Target.APIKeyEnv != "PROMPTKIT_MISSING_KEY" {
t.Fatalf("expected api_key_env name to remain on target, got %q", llmClient.lastReq.Target.APIKeyEnv)
}

View File

@@ -45,6 +45,101 @@ func (v *FSValidator) Validate(ctx context.Context, artifact *domain.Artifact, c
return validateArtifact(ctx, artifact, contract, v.validateJSONSchema)
}
type preparedValidation struct {
contract domain.OutputContract
schemaDocument any
schema *jsonschema.Schema
}
func (p *preparedValidation) Validate(ctx context.Context, artifact *domain.Artifact) (domain.ValidationResult, error) {
return validateArtifact(ctx, artifact, p.contract, p.validateJSONSchema)
}
func (p *preparedValidation) SchemaDocument() any {
return p.schemaDocument
}
func (p *preparedValidation) validateJSONSchema(instance any, _ string) ([]string, error) {
if p.schema == nil {
return nil, errors.New("prepared JSON schema is unavailable")
}
if err := p.schema.Validate(instance); err != nil {
return []string{fmt.Sprintf("json schema validation failed: %v", err)}, nil
}
return nil, nil
}
func (v *StandardValidator) PrepareValidation(ctx context.Context, contract domain.OutputContract) (PreparedValidation, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
prepared := &preparedValidation{contract: contract}
if contract.ValidationMode != domain.ValidationJSONSchema {
return prepared, nil
}
resolvedSchemaPath, err := v.resolveSchemaPath(contract.SchemaPath)
if err != nil {
return nil, err
}
schemaDocument, err := loadJSONSchemaFile(resolvedSchemaPath)
if err != nil {
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", resolvedSchemaPath, err)
}
schemaRoot, err := v.schemaRoot()
if err != nil {
return nil, err
}
compiler := newSchemaCompiler(standardSchemaLoader{root: schemaRoot})
if err := compiler.AddResource(resolvedSchemaPath, schemaDocument); err != nil {
return nil, fmt.Errorf("failed to register JSON schema %q: %w", resolvedSchemaPath, err)
}
schema, err := compiler.Compile(resolvedSchemaPath)
if err != nil {
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", resolvedSchemaPath, err)
}
if err := ctx.Err(); err != nil {
return nil, err
}
prepared.schemaDocument = schemaDocument
prepared.schema = schema
return prepared, nil
}
func (v *FSValidator) PrepareValidation(ctx context.Context, contract domain.OutputContract) (PreparedValidation, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
prepared := &preparedValidation{contract: contract}
if contract.ValidationMode != domain.ValidationJSONSchema {
return prepared, nil
}
schemaName, schemaDocument, err := v.loadSchemaDocument(contract.SchemaPath)
if err != nil {
return nil, err
}
resourceURL := fsSchemaResourceURL(schemaName)
compiler := newSchemaCompiler(fsSchemaLoader{fsys: v.fsys, root: filecatalog.CleanFSRoot(v.root)})
if err := compiler.AddResource(resourceURL, schemaDocument); err != nil {
return nil, fmt.Errorf("failed to register JSON schema %q: %w", schemaName, err)
}
schema, err := compiler.Compile(resourceURL)
if err != nil {
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", schemaName, err)
}
if err := ctx.Err(); err != nil {
return nil, err
}
prepared.schemaDocument = schemaDocument
prepared.schema = schema
return prepared, nil
}
type schemaValidatorFunc func(instance any, schemaPath string) ([]string, error)
func validateArtifact(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract, validateSchema schemaValidatorFunc) (domain.ValidationResult, error) {

View File

@@ -5,6 +5,7 @@ import (
"encoding/json"
"os"
"path/filepath"
"reflect"
"strconv"
"strings"
"testing"
@@ -120,6 +121,68 @@ func TestStandardValidatorJSONSchemaSuccess(t *testing.T) {
}
}
func TestStandardValidatorPreparedSchemaSurvivesSourceRemoval(t *testing.T) {
tmp := t.TempDir()
rootSchema := []byte(`{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "original root",
"type": "object",
"required": ["value"],
"properties": {
"value": {"$ref": "value.json"}
}
}`)
rootPath := filepath.Join(tmp, "schema.json")
referencePath := filepath.Join(tmp, "value.json")
if err := os.WriteFile(rootPath, rootSchema, 0o644); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(referencePath, []byte(`{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "integer",
"minimum": 2
}`), 0o644); err != nil {
t.Fatal(err)
}
validator := NewStandardValidator(tmp)
preparer, ok := validator.(ValidationPreparer)
if !ok {
t.Fatal("standard validator does not support validation preparation")
}
prepared, err := preparer.PrepareValidation(context.Background(), domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "schema.json",
})
if err != nil {
t.Fatalf("prepare validation: %v", err)
}
assertSchemaDocument(t, prepared.SchemaDocument(), rootSchema)
if err := os.Remove(rootPath); err != nil {
t.Fatal(err)
}
if err := os.Remove(referencePath); err != nil {
t.Fatal(err)
}
valid, err := prepared.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"value":3}`)})
if err != nil {
t.Fatalf("validate prepared artifact: %v", err)
}
if valid.Status != domain.ValidationPassed || !valid.IsValid {
t.Fatalf("expected passed/valid, got status=%q valid=%v errors=%v", valid.Status, valid.IsValid, valid.Errors)
}
invalid, err := prepared.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"value":"changed"}`)})
if err != nil {
t.Fatalf("validate prepared artifact: %v", err)
}
if invalid.Status != domain.ValidationFailed || invalid.IsValid {
t.Fatalf("expected failed/invalid, got status=%q valid=%v", invalid.Status, invalid.IsValid)
}
}
func TestStandardValidatorJSONSchemaNestedSchemaPathSuccess(t *testing.T) {
tmp := t.TempDir()
nestedDir := filepath.Join(tmp, "dnd")
@@ -295,6 +358,64 @@ func TestFSValidatorJSONSchemaSuccess(t *testing.T) {
}
}
func TestFSValidatorPreparedSchemaSurvivesSourceMutation(t *testing.T) {
rootSchema := []byte(`{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "original root",
"type": "object",
"required": ["value"],
"properties": {
"value": {"$ref": "value.json"}
}
}`)
fsys := fstest.MapFS{
"schema.json": &fstest.MapFile{Data: rootSchema},
"value.json": &fstest.MapFile{Data: []byte(`{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "integer",
"minimum": 2
}`)},
}
validator := NewFSValidator(fsys, ".")
preparer, ok := validator.(ValidationPreparer)
if !ok {
t.Fatal("filesystem validator does not support validation preparation")
}
prepared, err := preparer.PrepareValidation(context.Background(), domain.OutputContract{
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "schema.json",
})
if err != nil {
t.Fatalf("prepare validation: %v", err)
}
assertSchemaDocument(t, prepared.SchemaDocument(), rootSchema)
fsys["schema.json"] = &fstest.MapFile{Data: []byte(`{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "string"
}`)}
fsys["value.json"] = &fstest.MapFile{Data: []byte(`{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "string"
}`)}
valid, err := prepared.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"value":3}`)})
if err != nil {
t.Fatalf("validate prepared artifact: %v", err)
}
if valid.Status != domain.ValidationPassed || !valid.IsValid {
t.Fatalf("expected passed/valid, got status=%q valid=%v errors=%v", valid.Status, valid.IsValid, valid.Errors)
}
invalid, err := prepared.Validate(context.Background(), &domain.Artifact{Body: []byte(`{"value":"changed"}`)})
if err != nil {
t.Fatalf("validate prepared artifact: %v", err)
}
if invalid.Status != domain.ValidationFailed || invalid.IsValid {
t.Fatalf("expected failed/invalid, got status=%q valid=%v", invalid.Status, invalid.IsValid)
}
}
func TestFSValidatorJSONSchemaRegistrationError(t *testing.T) {
v := NewFSValidator(fstest.MapFS{
"schemas/%zz.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)},
@@ -548,3 +669,15 @@ func TestJSONSchemaDialectIsDraft2020(t *testing.T) {
})
}
}
func assertSchemaDocument(t *testing.T, got any, expectedJSON []byte) {
t.Helper()
var expected any
if err := json.Unmarshal(expectedJSON, &expected); err != nil {
t.Fatalf("decode expected schema document: %v", err)
}
if !reflect.DeepEqual(got, expected) {
t.Fatalf("schema document mismatch:\n got: %#v\nwant: %#v", got, expected)
}
}

View File

@@ -2,6 +2,7 @@ package validate
import (
"context"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
@@ -10,6 +11,20 @@ type Validator interface {
Validate(ctx context.Context, artifact *domain.Artifact, contract domain.OutputContract) (domain.ValidationResult, error)
}
// PreparedValidation validates artifacts against one frozen output contract.
type PreparedValidation interface {
Validate(ctx context.Context, artifact *domain.Artifact) (domain.ValidationResult, error)
// SchemaDocument returns the root JSON Schema document used for provider
// structured output, or nil for non-schema modes. Returned internal
// immutable state must not be mutated.
SchemaDocument() any
}
// ValidationPreparer freezes validation resources for one output contract.
type ValidationPreparer interface {
PrepareValidation(ctx context.Context, contract domain.OutputContract) (PreparedValidation, error)
}
// SchemaDocumentLoader loads JSON schema documents using validator path semantics.
type SchemaDocumentLoader interface {
LoadSchemaDocument(ctx context.Context, schemaPath string) (any, error)

55
prepared_execution.go Normal file
View File

@@ -0,0 +1,55 @@
package promptkit
import "gitea.maximumdirect.net/eric/promptkit/internal/usecase"
const preparedExecutionString = "promptkit.PreparedExecution{opaque}"
// PreparedExecution is an opaque, in-process handle for one completely
// prepared execution. A handle is bound to the [Engine] that created it and
// permits one [Engine.RunPrepared] invocation.
//
// PreparedExecution contains no supported serializable state and cannot be
// used as a restartable job. Copying the value preserves the same shared
// lifecycle; it does not create another execution attempt.
type PreparedExecution struct {
internal *usecase.PreparedExecution
}
// Details returns a fresh caller-owned, credential-redacted copy of the
// prepared request details. Mutating the result cannot affect execution or a
// later Details call. Details remains available after execution or discard.
//
// A nil receiver or zero-value PreparedExecution returns a zero [PreparedRun].
func (p *PreparedExecution) Details() PreparedRun {
if p == nil || p.internal == nil {
return PreparedRun{}
}
details := fromDomainPreparedRun(p.internal.Details())
if details == nil {
return PreparedRun{}
}
return *details
}
// Discard invalidates an unclaimed handle and drops Promptkit's references to
// its execution-only state. Discard is nil-safe and idempotent. It does not
// cancel an execution that has already claimed the handle; use the
// [Engine.RunPrepared] context for cancellation.
func (p *PreparedExecution) Discard() {
if p == nil || p.internal == nil {
return
}
p.internal.Discard()
}
// String returns a constant representation that exposes no retained request,
// rendered content, or credential data.
func (p *PreparedExecution) String() string {
return preparedExecutionString
}
// GoString returns a constant Go-syntax representation that exposes no
// retained request, rendered content, or credential data.
func (p *PreparedExecution) GoString() string {
return preparedExecutionString
}

View File

@@ -0,0 +1,773 @@
package promptkit_test
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"reflect"
"strings"
"sync"
"testing"
"testing/fstest"
"time"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestPreparedExecutionFreezesSourcesAndReturnsIndependentDetails(t *testing.T) {
promptSource := preparedPromptSource("original")
profileSource := preparedProfileSource("original-model")
schemaSource := preparedSchemaSource()
reader := &mutablePreparedArtifactReader{
body: "original artifact",
hash: "original-input-hash",
}
client := &preparedRecordingClient{
response: &promptkit.GenerateResponse{Content: `{"value":3}`},
}
engine, err := promptkit.NewEngine(
promptkit.Config{},
promptkit.WithPromptFS(promptSource, "."),
promptkit.WithProfileFS(profileSource, "."),
promptkit.WithSchemaFS(schemaSource, "."),
promptkit.WithArtifactReader(reader),
promptkit.WithLLMClient(client),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
temperature := 0.25
extraParams := map[string]any{
"nested": map[string]any{"source": "original"},
}
request := promptkit.RunRequest{
PromptID: "prepared",
Inputs: map[string]promptkit.ArtifactRef{
"input": promptkit.Inline("original request input"),
},
Vars: map[string]string{"label": "original variable"},
Execution: &promptkit.ExecutionTargetOverride{
Temperature: &temperature,
ExtraParams: extraParams,
},
}
preparationContext, cancelPreparation := context.WithCancel(context.Background())
prepared, err := engine.PrepareExecution(preparationContext, request)
if err != nil {
t.Fatalf("prepare execution: %v", err)
}
cancelPreparation()
request.PromptID = "changed"
request.Inputs["input"] = promptkit.Inline("changed request input")
request.Vars["label"] = "changed variable"
temperature = 1.5
extraParams["nested"].(map[string]any)["source"] = "changed"
promptSource["prompt.yaml"] = &fstest.MapFile{Data: []byte(`id: changed`)}
profileSource["profile.yaml"] = &fstest.MapFile{Data: []byte(`id: changed`)}
schemaSource["schema.json"] = &fstest.MapFile{Data: []byte(`{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "changed root",
"type": "string"
}`)}
schemaSource["value.json"] = &fstest.MapFile{Data: []byte(`{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "string"
}`)}
reader.set("changed artifact", "changed-input-hash")
first := prepared.Details()
first.Messages[0].Content = "changed details"
first.InputHashes["input"] = "changed-details-hash"
first.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["source"] = "changed details"
first.StructuredOutput.JSONSchema.Schema.(map[string]any)["title"] = "changed details"
second := prepared.Details()
if second.Messages[0].Content != "Input=original artifact Label=original variable" {
t.Fatalf("details message changed: %q", second.Messages[0].Content)
}
if second.InputHashes["input"] != "original-input-hash" {
t.Fatalf("details input hash changed: %q", second.InputHashes["input"])
}
if second.EffectiveModelParams.Model != "original-model" ||
second.EffectiveModelParams.Temperature != 0.25 ||
second.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["source"] != "original" {
t.Fatalf("details target changed: %+v", second.EffectiveModelParams)
}
schema := second.StructuredOutput.JSONSchema.Schema.(map[string]any)
if schema["title"] != "original root" {
t.Fatalf("details schema changed: %#v", schema)
}
result, err := engine.RunPrepared(context.Background(), prepared)
if err != nil {
t.Fatalf("run prepared after preparation-context cancellation: %v", err)
}
if result.Validation.Status != promptkit.ValidationPassed || !result.Validation.IsValid {
t.Fatalf("frozen schema did not validate original output: %+v", result.Validation)
}
if reader.callCount() != 1 {
t.Fatalf("execution reopened artifact source: calls=%d", reader.callCount())
}
requests := client.snapshot()
if len(requests) != 1 {
t.Fatalf("generation calls=%d, want 1", len(requests))
}
generated := requests[0]
if generated.Prompt.Messages[0].Content != second.Messages[0].Content ||
generated.Target.Model != second.EffectiveModelParams.Model ||
!reflect.DeepEqual(generated.Target.ExtraParams, second.EffectiveModelParams.ExtraParams) ||
!reflect.DeepEqual(generated.StructuredOutput, second.StructuredOutput) {
t.Fatalf("generation did not use frozen details:\nrequest=%+v\ndetails=%+v", generated, second)
}
if result.PromptID != second.PromptID ||
result.PromptVersion != second.PromptVersion ||
result.PromptHash != second.PromptHash ||
result.SessionID != second.SessionID ||
result.RenderedPromptHash != second.RenderedPromptHash ||
result.SelectedProfileID != second.SelectedProfileID ||
result.SelectedBackendID != second.SelectedBackendID ||
!reflect.DeepEqual(result.EffectiveModelParams, second.EffectiveModelParams) ||
!reflect.DeepEqual(result.InputHashes, second.InputHashes) {
t.Fatalf("result provenance does not match details:\nresult=%+v\ndetails=%+v", result, second)
}
}
func TestPreparedExecutionLifecycleAndEngineBinding(t *testing.T) {
ownerClient := &preparedRecordingClient{
response: &promptkit.GenerateResponse{Content: "ok"},
}
owner := newPreparedContractEngine(t, ownerClient, "owner content")
foreign := newPreparedContractEngine(t, &preparedRecordingClient{
response: &promptkit.GenerateResponse{Content: "unexpected"},
}, "foreign content")
prepared, err := owner.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
if err != nil {
t.Fatalf("prepare execution: %v", err)
}
copied := *prepared
var nilEngine *promptkit.Engine
if result, err := nilEngine.RunPrepared(context.Background(), prepared); result != nil ||
!errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("nil engine result=(%+v, %v), want ErrInvalidConfig", result, err)
}
if result, err := foreign.RunPrepared(context.Background(), prepared); result != nil ||
!errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("foreign engine result=(%+v, %v), want ErrInvalidRequest", result, err)
}
if result, err := owner.RunPrepared(context.Background(), nil); result != nil ||
!errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("nil handle result=(%+v, %v), want ErrInvalidRequest", result, err)
}
if result, err := owner.RunPrepared(context.Background(), &promptkit.PreparedExecution{}); result != nil ||
!errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("zero handle result=(%+v, %v), want ErrInvalidRequest", result, err)
}
result, err := owner.RunPrepared(context.Background(), &copied)
if err != nil || result == nil {
t.Fatalf("owner run prepared=(%+v, %v), want success", result, err)
}
for name, handle := range map[string]*promptkit.PreparedExecution{
"original": prepared,
"copy": &copied,
} {
if result, err := owner.RunPrepared(context.Background(), handle); result != nil ||
!errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("%s reused handle result=(%+v, %v), want ErrInvalidRequest", name, result, err)
}
if handle.Details().PromptID != "prepared" {
t.Fatalf("%s details unavailable after execution", name)
}
}
if len(ownerClient.snapshot()) != 1 {
t.Fatalf("owner generation calls=%d, want 1", len(ownerClient.snapshot()))
}
collaboratorFailure := errors.New("prepared collaborator failure")
failingClient := &preparedRecordingClient{err: collaboratorFailure}
failingEngine := newPreparedContractEngine(t, failingClient, "failure content")
failing, err := failingEngine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
if err != nil {
t.Fatalf("prepare failing execution: %v", err)
}
if result, err := failingEngine.RunPrepared(context.Background(), failing); result != nil ||
!errors.Is(err, promptkit.ErrLLMGenerate) ||
!errors.Is(err, collaboratorFailure) {
t.Fatalf("generation failure result=(%+v, %v), want public and collaborator identities", result, err)
}
if result, err := failingEngine.RunPrepared(context.Background(), failing); result != nil ||
!errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("failed execution was reusable: result=(%+v, %v)", result, err)
}
cancellationRelease := make(chan struct{})
cancellationStarted := make(chan struct{}, 1)
cancelingEngine := newPreparedContractEngine(t, &preparedRecordingClient{
response: &promptkit.GenerateResponse{Content: "unexpected"},
started: cancellationStarted,
release: cancellationRelease,
}, "cancellation content")
canceling, err := cancelingEngine.PrepareExecution(
context.Background(),
promptkit.RunRequest{PromptID: "prepared"},
)
if err != nil {
t.Fatalf("prepare canceled execution: %v", err)
}
executionContext, cancelExecution := context.WithCancel(context.Background())
type canceledOutcome struct {
result *promptkit.RunResult
err error
}
canceledResult := make(chan canceledOutcome, 1)
go func() {
result, runErr := cancelingEngine.RunPrepared(executionContext, canceling)
canceledResult <- canceledOutcome{result: result, err: runErr}
}()
select {
case <-cancellationStarted:
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for cancelable generation")
}
cancelExecution()
select {
case outcome := <-canceledResult:
if outcome.result != nil ||
!errors.Is(outcome.err, promptkit.ErrLLMGenerate) ||
!errors.Is(outcome.err, context.Canceled) {
t.Fatalf(
"canceled execution=(%+v, %v), want generation and context identities",
outcome.result,
outcome.err,
)
}
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for canceled execution")
}
if result, err := cancelingEngine.RunPrepared(context.Background(), canceling); result != nil ||
!errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("canceled execution was reusable: result=(%+v, %v)", result, err)
}
}
func TestPreparedExecutionConcurrentClaimAllowsOneGeneration(t *testing.T) {
release := make(chan struct{})
client := &preparedRecordingClient{
response: &promptkit.GenerateResponse{Content: "ok"},
started: make(chan struct{}, 1),
release: release,
}
engine := newPreparedContractEngine(t, client, "concurrent content")
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
if err != nil {
t.Fatalf("prepare execution: %v", err)
}
type outcome struct {
result *promptkit.RunResult
err error
}
outcomes := make(chan outcome, 2)
for i := 0; i < 2; i++ {
go func() {
result, runErr := engine.RunPrepared(context.Background(), prepared)
outcomes <- outcome{result: result, err: runErr}
}()
}
select {
case <-client.started:
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for generation")
}
select {
case loser := <-outcomes:
if loser.result != nil || !errors.Is(loser.err, promptkit.ErrInvalidRequest) {
t.Fatalf("concurrent loser=(%+v, %v), want ErrInvalidRequest", loser.result, loser.err)
}
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for rejected concurrent claim")
}
close(release)
select {
case winner := <-outcomes:
if winner.err != nil || winner.result == nil {
t.Fatalf("concurrent winner=(%+v, %v), want success", winner.result, winner.err)
}
case <-time.After(5 * time.Second):
t.Fatal("timed out waiting for successful concurrent claim")
}
if len(client.snapshot()) != 1 {
t.Fatalf("generation calls=%d, want 1", len(client.snapshot()))
}
}
func TestPreparedExecutionRunAndDiscardRaceHasOneWinner(t *testing.T) {
const attempts = 32
for i := 0; i < attempts; i++ {
client := &preparedRecordingClient{
response: &promptkit.GenerateResponse{Content: "ok"},
}
engine := newPreparedContractEngine(t, client, "race content")
prepared, err := engine.PrepareExecution(
context.Background(),
promptkit.RunRequest{PromptID: "prepared"},
)
if err != nil {
t.Fatalf("attempt %d prepare execution: %v", i, err)
}
start := make(chan struct{})
type outcome struct {
result *promptkit.RunResult
err error
}
runOutcome := make(chan outcome, 1)
discardDone := make(chan struct{})
go func() {
<-start
result, runErr := engine.RunPrepared(context.Background(), prepared)
runOutcome <- outcome{result: result, err: runErr}
}()
go func() {
<-start
prepared.Discard()
close(discardDone)
}()
close(start)
runResult := <-runOutcome
<-discardDone
calls := len(client.snapshot())
switch {
case runResult.err == nil:
if runResult.result == nil || calls != 1 {
t.Fatalf(
"attempt %d run won with outcome=(%+v, %v), generation calls=%d",
i,
runResult.result,
runResult.err,
calls,
)
}
case errors.Is(runResult.err, promptkit.ErrInvalidRequest):
if runResult.result != nil || calls != 0 {
t.Fatalf(
"attempt %d discard won with outcome=(%+v, %v), generation calls=%d",
i,
runResult.result,
runResult.err,
calls,
)
}
default:
t.Fatalf("attempt %d unexpected run outcome=(%+v, %v)", i, runResult.result, runResult.err)
}
if prepared.Details().PromptID != "prepared" {
t.Fatalf("attempt %d details unavailable after race", i)
}
}
}
func TestPreparedExecutionDiscardAndFormattingDoNotExposePrivateState(t *testing.T) {
const (
directCredential = "pk-test-direct-credential-41f7"
renderedContent = "rendered-content-sentinel-98d2"
)
client := &preparedRecordingClient{
response: &promptkit.GenerateResponse{Content: "generated output"},
}
engine, err := promptkit.NewEngine(
promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prepared", "profile", renderedContent), "."),
promptkit.WithProfiles(promptkit.Profile{
ID: "profile",
Endpoint: "http://example.test/v1",
Model: "model",
APIKeyRequired: true,
}),
promptkit.WithLLMClient(client),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
PromptID: "prepared",
APIKey: directCredential,
})
if err != nil {
t.Fatalf("prepare execution: %v", err)
}
formattedValues := []string{
fmt.Sprint(prepared),
fmt.Sprintf("%+v", prepared),
fmt.Sprintf("%#v", prepared),
}
for _, formatted := range formattedValues {
if formatted != "promptkit.PreparedExecution{opaque}" {
t.Fatalf("unexpected opaque formatting: %q", formatted)
}
assertPreparedPrivateValuesAbsent(t, formatted, directCredential, renderedContent)
}
payload, err := json.Marshal(prepared)
if err != nil {
t.Fatalf("marshal opaque handle: %v", err)
}
assertPreparedPrivateValuesAbsent(t, string(payload), directCredential, renderedContent)
detailsBefore := prepared.Details()
detailsJSON, err := json.Marshal(detailsBefore)
if err != nil {
t.Fatalf("marshal prepared details: %v", err)
}
assertPreparedPrivateValuesAbsent(t, string(detailsJSON), directCredential)
prepared.Discard()
prepared.Discard()
result, lifecycleErr := engine.RunPrepared(context.Background(), prepared)
if result != nil || !errors.Is(lifecycleErr, promptkit.ErrInvalidRequest) {
t.Fatalf("discarded execution result=(%+v, %v), want ErrInvalidRequest", result, lifecycleErr)
}
assertPreparedPrivateValuesAbsent(t, lifecycleErr.Error(), directCredential, renderedContent)
if !reflect.DeepEqual(prepared.Details(), detailsBefore) {
t.Fatal("details changed after discard")
}
executed, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{
PromptID: "prepared",
APIKey: directCredential,
})
if err != nil {
t.Fatalf("prepare execution for request inspection: %v", err)
}
executionResult, err := engine.RunPrepared(context.Background(), executed)
if err != nil {
t.Fatalf("run execution for request inspection: %v", err)
}
requests := client.snapshot()
if len(requests) != 1 || requests[0].APIKey != directCredential {
t.Fatalf("direct credential did not reach only the client credential field: %#v", requests)
}
requestJSON, err := json.Marshal(requests[0])
if err != nil {
t.Fatalf("marshal captured generate request: %v", err)
}
for _, value := range []string{
fmt.Sprint(requests[0]),
fmt.Sprintf("%+v", requests[0]),
fmt.Sprintf("%#v", requests[0]),
string(requestJSON),
fmt.Sprint(executionResult),
} {
assertPreparedPrivateValuesAbsent(t, value, directCredential)
}
resultJSON, err := json.Marshal(executionResult)
if err != nil {
t.Fatalf("marshal execution result: %v", err)
}
assertPreparedPrivateValuesAbsent(t, string(resultJSON), directCredential)
var nilHandle *promptkit.PreparedExecution
nilHandle.Discard()
if !reflect.DeepEqual(nilHandle.Details(), promptkit.PreparedRun{}) {
t.Fatalf("nil handle details=%+v, want zero value", nilHandle.Details())
}
zeroHandle := &promptkit.PreparedExecution{}
zeroHandle.Discard()
if !reflect.DeepEqual(zeroHandle.Details(), promptkit.PreparedRun{}) {
t.Fatalf("zero handle details=%+v, want zero value", zeroHandle.Details())
}
}
func TestPreparedExecutionCredentialCapacityAndTimingBoundaries(t *testing.T) {
t.Run("credential is rechecked before generation", func(t *testing.T) {
const (
environmentName = "PROMPTKIT_PREPARED_CONTRACT_KEY"
environmentKey = "environment-credential-sentinel"
)
t.Setenv(environmentName, environmentKey)
client := &preparedRecordingClient{
response: &promptkit.GenerateResponse{Content: "unexpected"},
}
engine, err := promptkit.NewEngine(
promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prepared", "profile", "content"), "."),
promptkit.WithProfileFS(preparedCredentialProfileSource(environmentName), "."),
promptkit.WithLLMClient(client),
)
if err != nil {
t.Fatalf("construct credential engine: %v", err)
}
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
if err != nil {
t.Fatalf("prepare credential execution: %v", err)
}
if err := os.Unsetenv(environmentName); err != nil {
t.Fatalf("unset credential environment: %v", err)
}
result, err := engine.RunPrepared(context.Background(), prepared)
if result != nil ||
!errors.Is(err, promptkit.ErrInvalidRequest) ||
!errors.Is(err, promptkit.ErrAPIKeyEnvMissing) {
t.Fatalf("credential execution=(%+v, %v), want credential identities", result, err)
}
if len(client.snapshot()) != 0 {
t.Fatalf("credential failure reached generation: %d calls", len(client.snapshot()))
}
assertPreparedPrivateValuesAbsent(t, err.Error(), environmentKey)
if result, err := engine.RunPrepared(context.Background(), prepared); result != nil ||
!errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("credential failure did not consume handle: result=(%+v, %v)", result, err)
}
})
t.Run("preparation does not admit and execution timing starts after retention", func(t *testing.T) {
release := make(chan struct{})
client := newCapacityGateClient(release, 4)
engine := newBackendCapacityEngine(t, client, 1, capacityInt(0), nil)
activeRun := make(chan capacityRunResult, 1)
go runCapacityRequest(
engine,
context.Background(),
promptkit.RunRequest{PromptID: "prompt"},
activeRun,
)
awaitCapacityRequest(t, client.started)
prepared, err := engine.PrepareExecution(
context.Background(),
promptkit.RunRequest{PromptID: "prompt"},
)
if err != nil {
t.Fatalf("prepare while capacity is full: %v", err)
}
if _, _, calls := client.snapshot(); calls != 1 {
t.Fatalf("preparation invoked generation: calls=%d", calls)
}
if result, err := engine.RunPrepared(context.Background(), prepared); result != nil ||
!errors.Is(err, promptkit.ErrCapacityExceeded) {
t.Fatalf("capacity execution=(%+v, %v), want ErrCapacityExceeded", result, err)
} else {
var capacityErr *promptkit.CapacityError
if !errors.As(err, &capacityErr) || capacityErr == nil || capacityErr.BackendID != "limited" {
t.Fatalf("capacity execution=%v, want limited CapacityError", err)
}
}
if result, err := engine.RunPrepared(context.Background(), prepared); result != nil ||
!errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("capacity rejection did not consume handle: result=(%+v, %v)", result, err)
}
if prepared.Details().PromptID != "prompt" {
t.Fatal("details unavailable after capacity rejection")
}
close(release)
activeOutcome := awaitCapacityRun(t, activeRun)
if activeOutcome.err != nil || activeOutcome.result == nil {
t.Fatalf("active run outcome=(%+v, %v), want success", activeOutcome.result, activeOutcome.err)
}
timed, err := engine.PrepareExecution(
context.Background(),
promptkit.RunRequest{PromptID: "prompt"},
)
if err != nil {
t.Fatalf("prepare timed execution: %v", err)
}
details := timed.Details()
time.Sleep(25 * time.Millisecond)
executionFloor := time.Now().UTC()
result, err := engine.RunPrepared(context.Background(), timed)
if err != nil {
t.Fatalf("run timed execution: %v", err)
}
if result.StartTime.Before(executionFloor) ||
!result.StartTime.After(details.EndTime) ||
result.EndTime.Before(result.StartTime) ||
result.Duration != result.EndTime.Sub(result.StartTime) {
t.Fatalf(
"execution timing includes preparation or retention: details_end=%s floor=%s result=%+v",
details.EndTime,
executionFloor,
result,
)
}
if _, _, calls := client.snapshot(); calls != 2 {
t.Fatalf("generation calls=%d, want active and timed executions only", calls)
}
})
}
type mutablePreparedArtifactReader struct {
mu sync.Mutex
body string
hash string
calls int
}
func (r *mutablePreparedArtifactReader) Read(
_ context.Context,
_ promptkit.ArtifactRef,
) (*promptkit.Artifact, error) {
r.mu.Lock()
defer r.mu.Unlock()
r.calls++
return &promptkit.Artifact{
Body: []byte(r.body),
Hash: r.hash,
}, nil
}
func (r *mutablePreparedArtifactReader) set(body, hash string) {
r.mu.Lock()
defer r.mu.Unlock()
r.body = body
r.hash = hash
}
func (r *mutablePreparedArtifactReader) callCount() int {
r.mu.Lock()
defer r.mu.Unlock()
return r.calls
}
type preparedRecordingClient struct {
mu sync.Mutex
response *promptkit.GenerateResponse
err error
requests []promptkit.GenerateRequest
started chan struct{}
release <-chan struct{}
}
func (c *preparedRecordingClient) Generate(
ctx context.Context,
request promptkit.GenerateRequest,
) (*promptkit.GenerateResponse, error) {
c.mu.Lock()
c.requests = append(c.requests, request)
c.mu.Unlock()
if c.started != nil {
c.started <- struct{}{}
}
if c.release != nil {
select {
case <-c.release:
case <-ctx.Done():
return nil, ctx.Err()
}
}
if c.err != nil {
return nil, c.err
}
return c.response, nil
}
func (c *preparedRecordingClient) snapshot() []promptkit.GenerateRequest {
c.mu.Lock()
defer c.mu.Unlock()
return append([]promptkit.GenerateRequest(nil), c.requests...)
}
func newPreparedContractEngine(
t *testing.T,
client promptkit.LLMClient,
message string,
) *promptkit.Engine {
t.Helper()
engine, err := promptkit.NewEngine(
promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prepared", "profile", message), "."),
promptkit.WithProfiles(promptkit.Profile{
ID: "profile",
Endpoint: "http://example.test/v1",
Model: "model",
}),
promptkit.WithLLMClient(client),
)
if err != nil {
t.Fatalf("construct prepared execution engine: %v", err)
}
return engine
}
func preparedPromptSource(label string) fstest.MapFS {
return fstest.MapFS{
"prompt.yaml": &fstest.MapFile{Data: []byte(`id: prepared
version: "1"
default_profile: profile
inputs:
- name: input
required: true
messages:
- role: user
content: 'Input={{input "input"}} Label={{.label}}'
description: ` + label + `
output:
format: json
validation_mode: json_schema
schema_path: schema.json
`)},
}
}
func preparedProfileSource(model string) fstest.MapFS {
return fstest.MapFS{
"profile.yaml": &fstest.MapFile{Data: []byte(`id: profile
endpoint: http://example.test/v1
model: ` + model + `
`)},
}
}
func preparedCredentialProfileSource(environmentName string) fstest.MapFS {
return fstest.MapFS{
"profile.yaml": &fstest.MapFile{Data: []byte(`id: profile
endpoint: http://example.test/v1
model: model
api_key_env: ` + environmentName + `
`)},
}
}
func preparedSchemaSource() fstest.MapFS {
return fstest.MapFS{
"schema.json": &fstest.MapFile{Data: []byte(`{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"title": "original root",
"type": "object",
"required": ["value"],
"properties": {
"value": {"$ref": "value.json"}
}
}`)},
"value.json": &fstest.MapFile{Data: []byte(`{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "integer",
"minimum": 2
}`)},
}
}
func assertPreparedPrivateValuesAbsent(t *testing.T, value string, privateValues ...string) {
t.Helper()
for _, privateValue := range privateValues {
if strings.Contains(value, privateValue) {
t.Fatalf("value exposed private data %q: %s", privateValue, value)
}
}
}

View File

@@ -5,6 +5,8 @@ import (
"encoding/json"
"errors"
"fmt"
"io/fs"
"reflect"
"strings"
"sync"
"sync/atomic"
@@ -15,6 +17,380 @@ import (
"gitea.maximumdirect.net/eric/promptkit"
)
type inspectionCountingFS struct {
opens atomic.Int64
}
func (f *inspectionCountingFS) Open(string) (fs.File, error) {
f.opens.Add(1)
return nil, fs.ErrNotExist
}
func TestInspectProfileResolvesCredentialStatesWithoutPromptOrGeneration(t *testing.T) {
const environmentName = "PROMPTKIT_INSPECTION_ABSENT_KEY"
t.Setenv(environmentName, "")
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}}
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(fstest.MapFS{}, "."),
promptkit.WithProfileFS(fstest.MapFS{
"environment.yaml": &fstest.MapFile{Data: []byte(`id: environment
endpoint: http://environment.example/v1
model: environment-model
api_key_env: PROMPTKIT_INSPECTION_ABSENT_KEY
`)},
}, "."),
promptkit.WithProfiles(
promptkit.Profile{ID: "direct", Endpoint: "http://direct.example/v1", Model: "direct-model", APIKeyRequired: true},
promptkit.Profile{ID: "none", Endpoint: "http://none.example/v1", Model: "none-model"},
),
promptkit.WithLLMClient(client),
)
if err != nil {
t.Fatalf("construct inspection engine: %v", err)
}
for _, tc := range []struct {
profileID string
wantEnv string
wantDirectKey bool
wantEndpoint string
}{
{profileID: " environment ", wantEnv: environmentName, wantEndpoint: "http://environment.example/v1"},
{profileID: "direct", wantDirectKey: true, wantEndpoint: "http://direct.example/v1"},
{profileID: "none", wantEndpoint: "http://none.example/v1"},
} {
t.Run(tc.profileID, func(t *testing.T) {
inspection, err := engine.InspectProfile(context.Background(), tc.profileID)
if err != nil {
t.Fatalf("inspect profile: %v", err)
}
if inspection.ProfileID != strings.TrimSpace(tc.profileID) ||
inspection.EffectiveModelParams.Endpoint != tc.wantEndpoint ||
inspection.EffectiveModelParams.BackendID != "" ||
inspection.EffectiveModelParams.APIKeyEnv != tc.wantEnv ||
inspection.APIKeyRequired != tc.wantDirectKey {
t.Fatalf("unexpected inspection: %#v", inspection)
}
})
}
if len(client.requests) != 0 {
t.Fatalf("inspection invoked the model client %d times", len(client.requests))
}
}
func TestInspectProfilePreservesPublicErrorIdentities(t *testing.T) {
newEngine := func(t *testing.T, options ...promptkit.Option) *promptkit.Engine {
t.Helper()
engine, err := promptkit.NewEngine(
promptkit.Config{},
append([]promptkit.Option{promptkit.WithPromptFS(fstest.MapFS{}, ".")}, options...)...,
)
if err != nil {
t.Fatalf("construct inspection engine: %v", err)
}
return engine
}
var nilEngine *promptkit.Engine
if result, err := nilEngine.InspectProfile(context.Background(), "profile"); result != nil ||
!errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("nil engine result=(%#v, %v), want ErrInvalidConfig", result, err)
}
valid := newEngine(t, promptkit.WithProfiles(promptkit.Profile{
ID: "profile", Endpoint: "http://profile.example/v1", Model: "model",
}))
if result, err := valid.InspectProfile(context.Background(), " \t "); result != nil ||
!errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("blank profile result=(%#v, %v), want ErrInvalidRequest", result, err)
}
if result, err := valid.InspectProfile(context.Background(), "missing"); result != nil ||
!errors.Is(err, promptkit.ErrProfileNotFound) || errors.Is(err, promptkit.ErrProfileLoad) {
t.Fatalf("missing profile result=(%#v, %v), want only ErrProfileNotFound", result, err)
}
malformed := newEngine(t, promptkit.WithProfileFS(fstest.MapFS{
"broken.yaml": &fstest.MapFile{Data: []byte("id: broken\nendpoint: http://broken.example/v1\nmodel: model\nunknown: value\n")},
}, "."))
if result, err := malformed.InspectProfile(context.Background(), "broken"); result != nil ||
!errors.Is(err, promptkit.ErrProfileLoad) {
t.Fatalf("malformed profile result=(%#v, %v), want ErrProfileLoad", result, err)
}
unknownBackend := newEngine(t, promptkit.WithProfiles(promptkit.Profile{
ID: "unknown-backend", BackendID: "unknown", Model: "model",
}))
if result, err := unknownBackend.InspectProfile(context.Background(), "unknown-backend"); result != nil ||
!errors.Is(err, promptkit.ErrProfileLoad) {
t.Fatalf("unknown backend result=(%#v, %v), want ErrProfileLoad", result, err)
}
countingFS := &inspectionCountingFS{}
canceled := newEngine(t, promptkit.WithProfileFS(countingFS, "."))
ctx, cancel := context.WithCancel(context.Background())
cancel()
if result, err := canceled.InspectProfile(ctx, "profile"); result != nil ||
!errors.Is(err, promptkit.ErrProfileLoad) || !errors.Is(err, context.Canceled) || countingFS.opens.Load() != 0 {
t.Fatalf("canceled inspection result=(%#v, %v), opens=%d", result, err, countingFS.opens.Load())
}
}
func TestInspectProfileReturnsIndependentTargetMatchingPreparation(t *testing.T) {
extraParams := map[string]any{
"nested": map[string]any{"value": "original"},
}
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
promptkit.WithProfiles(promptkit.Profile{
ID: "profile", Endpoint: "http://profile.example/v1", Model: "model", ExtraParams: extraParams,
}),
)
if err != nil {
t.Fatalf("construct inspection engine: %v", err)
}
first, err := engine.InspectProfile(context.Background(), "profile")
if err != nil {
t.Fatalf("first inspection: %v", err)
}
first.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["value"] = "changed"
first.EffectiveModelParams.ExtraParams["later"] = true
second, err := engine.InspectProfile(context.Background(), "profile")
if err != nil {
t.Fatalf("second inspection: %v", err)
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatalf("prepare after inspection mutation: %v", err)
}
for _, target := range []promptkit.ExecutionTarget{second.EffectiveModelParams, prepared.EffectiveModelParams} {
if target.ExtraParams["nested"].(map[string]any)["value"] != "original" || target.ExtraParams["later"] != nil {
t.Fatalf("inspection mutation reached engine-owned target: %#v", target.ExtraParams)
}
}
if !reflect.DeepEqual(second.EffectiveModelParams, prepared.EffectiveModelParams) {
t.Fatalf("inspection target=%#v, preparation target=%#v", second.EffectiveModelParams, prepared.EffectiveModelParams)
}
}
func TestInspectPromptReturnsDeclaredMetadataWithoutExecutionWork(t *testing.T) {
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}}
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(fstest.MapFS{
"report-v1.yaml": &fstest.MapFile{Data: []byte(`id: report
version: "1.0.0"
messages:
- role: user
content: old report
output:
format: text
validation_mode: none
`)},
"report-v2.yaml": &fstest.MapFile{Data: []byte(`id: report
version: "2.0.0"
default_profile: missing-profile
inputs:
- name: location
required: true
content_type: text/plain
description: Forecast location.
- name: units
content_type: text/plain
description: Unit preference.
messages:
- role: user
content_file: messages/report.md
output:
format: json
validation_mode: json_schema
schema_path: schemas/report.json
`)},
"messages/report.md": &fstest.MapFile{Data: []byte("rendered report body is not returned")},
}, "."),
promptkit.WithLLMClient(client),
)
if err != nil {
t.Fatalf("construct prompt-inspection engine: %v", err)
}
inspection, err := engine.InspectPrompt(context.Background(), "report", "2.0.0")
if err != nil {
t.Fatalf("inspect prompt: %v", err)
}
if inspection.PromptID != "report" ||
inspection.PromptVersion != "2.0.0" ||
inspection.PromptHash == "" ||
inspection.DefaultProfileID != "missing-profile" ||
inspection.OutputContract != (promptkit.OutputContract{
Format: promptkit.FormatJSON,
ValidationMode: promptkit.ValidationJSONSchema,
SchemaPath: "schemas/report.json",
}) {
t.Fatalf("unexpected inspection metadata: %#v", inspection)
}
wantInputs := []promptkit.PromptInputDefinition{
{Name: "location", Required: true, ContentType: "text/plain", Description: "Forecast location."},
{Name: "units", ContentType: "text/plain", Description: "Unit preference."},
}
if !reflect.DeepEqual(inspection.Inputs, wantInputs) {
t.Fatalf("inspection inputs=%#v, want %#v", inspection.Inputs, wantInputs)
}
if len(client.requests) != 0 {
t.Fatalf("inspection invoked the model client %d times", len(client.requests))
}
}
func TestInspectPromptPreservesPublicErrorIdentities(t *testing.T) {
newEngine := func(t *testing.T, source fstest.MapFS) *promptkit.Engine {
t.Helper()
engine, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithPromptFS(source, "."))
if err != nil {
t.Fatalf("construct prompt-inspection engine: %v", err)
}
return engine
}
validSource := fstest.MapFS{
"prompt.yaml": &fstest.MapFile{Data: []byte(`id: prompt
version: "1"
messages:
- role: user
content: body
output:
format: text
validation_mode: none
`)},
}
var nilEngine *promptkit.Engine
if result, err := nilEngine.InspectPrompt(context.Background(), "prompt", "1"); result != nil ||
!errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("nil engine result=(%#v, %v), want ErrInvalidConfig", result, err)
}
valid := newEngine(t, validSource)
if result, err := valid.InspectPrompt(context.Background(), " \t ", "1"); result != nil ||
!errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("blank prompt result=(%#v, %v), want ErrInvalidRequest", result, err)
}
if result, err := valid.InspectPrompt(context.Background(), "missing", "1"); result != nil ||
!errors.Is(err, promptkit.ErrPromptNotFound) || errors.Is(err, promptkit.ErrPromptLoad) {
t.Fatalf("missing prompt result=(%#v, %v), want only ErrPromptNotFound", result, err)
}
if result, err := valid.InspectPrompt(context.Background(), "prompt", "missing"); result != nil ||
!errors.Is(err, promptkit.ErrPromptNotFound) || errors.Is(err, promptkit.ErrPromptLoad) {
t.Fatalf("missing version result=(%#v, %v), want only ErrPromptNotFound", result, err)
}
ambiguous := newEngine(t, fstest.MapFS{
"one.yaml": &fstest.MapFile{Data: []byte(`id: prompt
version: "1"
messages:
- role: user
content: first
output:
format: text
validation_mode: none
`)},
"two.yaml": &fstest.MapFile{Data: []byte(`id: prompt
version: "2"
messages:
- role: user
content: second
output:
format: text
validation_mode: none
`)},
})
if result, err := ambiguous.InspectPrompt(context.Background(), "prompt", ""); result != nil ||
!errors.Is(err, promptkit.ErrPromptLoad) {
t.Fatalf("ambiguous prompt result=(%#v, %v), want ErrPromptLoad", result, err)
}
for name, source := range map[string]fstest.MapFS{
"malformed definition": {
"broken.yaml": &fstest.MapFile{Data: []byte("id: broken\nversion: \"1\"\nunknown: value\n")},
},
"missing content file": {
"broken.yaml": &fstest.MapFile{Data: []byte(`id: broken
version: "1"
messages:
- role: user
content_file: missing.md
output:
format: text
validation_mode: none
`)},
},
} {
t.Run(name, func(t *testing.T) {
if result, err := newEngine(t, source).InspectPrompt(context.Background(), "broken", "1"); result != nil ||
!errors.Is(err, promptkit.ErrPromptLoad) {
t.Fatalf("broken prompt result=(%#v, %v), want ErrPromptLoad", result, err)
}
})
}
countingFS := &inspectionCountingFS{}
canceled, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithPromptFS(countingFS, "."))
if err != nil {
t.Fatalf("construct canceled prompt-inspection engine: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if result, err := canceled.InspectPrompt(ctx, "prompt", "1"); result != nil ||
!errors.Is(err, promptkit.ErrPromptLoad) || !errors.Is(err, context.Canceled) || countingFS.opens.Load() != 0 {
t.Fatalf("canceled inspection result=(%#v, %v), opens=%d", result, err, countingFS.opens.Load())
}
}
func TestInspectPromptReturnsIndependentMetadataMatchingPreparation(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(fstest.MapFS{
"prompt.yaml": &fstest.MapFile{Data: []byte(`id: prompt
version: "1"
default_profile: profile
inputs:
- name: subject
content_type: text/plain
description: Summary subject.
messages:
- role: user
content: summarize
output:
format: markdown
validation_mode: basic
`)},
}, "."),
promptkit.WithProfiles(promptkit.Profile{
ID: "profile", Endpoint: "http://profile.example/v1", Model: "model",
}),
)
if err != nil {
t.Fatalf("construct prompt-inspection engine: %v", err)
}
first, err := engine.InspectPrompt(context.Background(), "prompt", "1")
if err != nil {
t.Fatalf("first inspection: %v", err)
}
first.Inputs[0].Name = "changed"
first.OutputContract.SchemaPath = "changed.json"
second, err := engine.InspectPrompt(context.Background(), "prompt", "1")
if err != nil {
t.Fatalf("second inspection: %v", err)
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt", PromptVersion: "1"})
if err != nil {
t.Fatalf("prepare after inspection mutation: %v", err)
}
if second.Inputs[0].Name != "subject" || second.OutputContract.SchemaPath != "" ||
prepared.OutputContract.SchemaPath != "" || second.PromptHash != prepared.PromptHash {
t.Fatalf("inspection mutation reached engine-owned prompt metadata: inspection=%#v prepared=%#v", second, prepared)
}
}
func TestPreparedRunJSONOmitsZeroTimingValues(t *testing.T) {
payload, err := json.Marshal(promptkit.PreparedRun{})
if err != nil {
@@ -138,6 +514,49 @@ func TestUnknownProfileBackendHasProfileLoadIdentity(t *testing.T) {
}
}
func TestLocalBackendConstructsAndRegistersConventionalBackend(t *testing.T) {
const (
localBackendID = "local"
limit = 2
)
if promptkit.BackendLocal != localBackendID {
t.Fatalf("BackendLocal=%q, want %q", promptkit.BackendLocal, localBackendID)
}
endpoint := "http://local.example/v1"
backend := promptkit.LocalBackend(endpoint, limit)
want := promptkit.Backend{
ID: localBackendID,
Endpoint: endpoint,
ConcurrencyLimit: limit,
}
if !reflect.DeepEqual(backend, want) {
t.Fatalf("LocalBackend()=%+v, want %+v", backend, want)
}
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "local-profile", "message"), "."),
promptkit.WithBackend(backend),
promptkit.WithProfiles(promptkit.Profile{
ID: "local-profile",
BackendID: localBackendID,
Model: "model",
}),
)
if err != nil {
t.Fatalf("construct engine with local backend: %v", err)
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatalf("prepare with local backend: %v", err)
}
if prepared.SelectedBackendID != localBackendID ||
prepared.EffectiveModelParams.Endpoint != endpoint {
t.Fatalf("unexpected local backend preparation: %+v", prepared)
}
}
func TestCustomBackendFlowsThroughProfilesOverridesAndInjectedClient(t *testing.T) {
t.Setenv("CUSTOM_LLM_KEY", "test-key")
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
@@ -554,6 +973,24 @@ func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) {
}
})
t.Run("fallback profile source", func(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
promptkit.WithFallbackProfileFS(contractProfileFS("profile", "first-model"), "."),
promptkit.WithFallbackProfileFS(contractProfileFS("profile", "second-model"), "."),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatalf("prepare from last fallback profile source: %v", err)
}
if prepared.EffectiveModelParams.Model != "second-model" {
t.Fatalf("expected last fallback profile source, got %q", prepared.EffectiveModelParams.Model)
}
})
t.Run("in-memory profiles", func(t *testing.T) {
first := profile
first.Model = "first-model"
@@ -638,6 +1075,212 @@ func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) {
})
}
func TestFallbackProfileSourcePrecedence(t *testing.T) {
const profileID = "application-profile"
prepareModel := func(t *testing.T, engine *promptkit.Engine, promptID string) string {
t.Helper()
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: promptID})
if err != nil {
t.Fatalf("prepare: %v", err)
}
return prepared.EffectiveModelParams.Model
}
t.Run("in-memory profiles override ordinary and fallback profiles", func(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
promptkit.WithProfileFS(contractProfileFS(profileID, "ordinary-model"), "."),
promptkit.WithProfiles(promptkit.Profile{ID: profileID, Endpoint: "http://example.test/v1", Model: "memory-model"}),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
if model := prepareModel(t, engine, "prompt"); model != "memory-model" {
t.Fatalf("expected in-memory profile, got %q", model)
}
})
t.Run("ordinary filesystem source overrides fallback profile", func(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
promptkit.WithProfileFS(contractProfileFS(profileID, "ordinary-model"), "."),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
if model := prepareModel(t, engine, "prompt"); model != "ordinary-model" {
t.Fatalf("expected ordinary profile, got %q", model)
}
})
t.Run("configured directory overrides fallback profile", func(t *testing.T) {
profileDir := t.TempDir()
writePublicProfileFile(t, profileDir, profileID, "http://example.test/v1", "directory-model")
engine, err := promptkit.NewEngine(promptkit.Config{ProfileDir: profileDir},
promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
if model := prepareModel(t, engine, "prompt"); model != "directory-model" {
t.Fatalf("expected configured directory profile, got %q", model)
}
})
t.Run("fallback profile overrides built-in profile", func(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."),
promptkit.WithFallbackProfileFS(contractProfileFS("mistral-small-3", "fallback-model"), "."),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
if model := prepareModel(t, engine, "prompt"); model != "fallback-model" {
t.Fatalf("expected fallback profile, got %q", model)
}
})
t.Run("missing fallback profile uses built-in profile", func(t *testing.T) {
t.Setenv("OPENROUTER_API_KEY", "test-key")
baseline, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."),
)
if err != nil {
t.Fatalf("construct baseline engine: %v", err)
}
want := prepareModel(t, baseline, "prompt")
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."),
promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
if model := prepareModel(t, engine, "prompt"); model != want {
t.Fatalf("expected built-in profile model %q, got %q", want, model)
}
})
}
func TestFallbackProfileSourcePreservesLazyLoadingAndErrors(t *testing.T) {
const profileID = "application-profile"
t.Run("construction defers malformed fallback profiles", func(t *testing.T) {
_, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(fstest.MapFS{}, "."),
promptkit.WithFallbackProfileFS(fstest.MapFS{
"broken.yaml": &fstest.MapFile{Data: []byte("id: broken\nunknown: value\n")},
}, "."),
)
if err != nil {
t.Fatalf("construct engine with malformed fallback profile: %v", err)
}
})
t.Run("unrelated malformed fallback profile does not block matching definition", func(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
promptkit.WithFallbackProfileFS(fstest.MapFS{
"broken.yaml": &fstest.MapFile{Data: []byte("id: unrelated\nunknown: value\n")},
"valid.yaml": &fstest.MapFile{Data: []byte("id: application-profile\nendpoint: http://example.test/v1\nmodel: fallback-model\n")},
}, "."),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatalf("prepare from valid fallback profile: %v", err)
}
if prepared.EffectiveModelParams.Model != "fallback-model" {
t.Fatalf("unexpected fallback profile model: %q", prepared.EffectiveModelParams.Model)
}
})
t.Run("matching malformed fallback profile does not reach built-in profile", func(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."),
promptkit.WithFallbackProfileFS(fstest.MapFS{
"mistral-small-3.yaml": &fstest.MapFile{Data: []byte("id: mistral-small-3\nendpoint: http://example.test/v1\nmodel: fallback-model\nunknown: value\n")},
}, "."),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
if _, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"}); !errors.Is(err, promptkit.ErrProfileLoad) {
t.Fatalf("expected ErrProfileLoad, got %v", err)
}
})
t.Run("matching malformed ordinary profile does not reach fallback profile", func(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
promptkit.WithProfileFS(fstest.MapFS{
"application-profile.yaml": &fstest.MapFile{Data: []byte("id: application-profile\nendpoint: http://example.test/v1\nmodel: ordinary-model\nunknown: value\n")},
}, "."),
promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
if _, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"}); !errors.Is(err, promptkit.ErrProfileLoad) {
t.Fatalf("expected ErrProfileLoad, got %v", err)
}
})
}
func TestFallbackProfileSourceWorksAcrossWorkflows(t *testing.T) {
const profileID = "application-profile"
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
promptkit.WithLLMClient(client),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
inspection, err := engine.InspectProfile(context.Background(), profileID)
if err != nil {
t.Fatalf("inspect fallback profile: %v", err)
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatalf("prepare fallback profile: %v", err)
}
preparedExecution, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatalf("prepare execution with fallback profile: %v", err)
}
preparedDetails := preparedExecution.Details()
preparedResult, err := engine.RunPrepared(context.Background(), preparedExecution)
if err != nil {
t.Fatalf("run prepared fallback profile: %v", err)
}
runResult, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatalf("run fallback profile: %v", err)
}
for name, model := range map[string]string{
"inspection": inspection.EffectiveModelParams.Model,
"preparation": prepared.EffectiveModelParams.Model,
"prepared execution": preparedDetails.EffectiveModelParams.Model,
"prepared result": preparedResult.EffectiveModelParams.Model,
"run result": runResult.EffectiveModelParams.Model,
} {
if model != "fallback-model" {
t.Fatalf("%s model=%q, want fallback-model", name, model)
}
}
}
func TestEngineSupportsConcurrentPrepareAndRun(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),

186
types.go
View File

@@ -51,7 +51,8 @@ const (
// ValidationPassed means the generated output satisfied its contract.
ValidationPassed ValidationStatus = "passed"
// ValidationFailed means validation completed and rejected the generated
// output. Engine.Run returns this status in a result, not as an error.
// output. Engine.Run and Engine.RunPrepared return this status in a result,
// not as an error.
ValidationFailed ValidationStatus = "failed"
// ValidationSkipped means ValidationNone selected no content check.
ValidationSkipped ValidationStatus = "skipped"
@@ -78,9 +79,10 @@ const (
// RunRequest selects one prompt execution. It has no stable JSON
// representation.
//
// Prepare and Run copy the request's maps, pointers, and nested
// JSON-compatible values before using them. The caller may mutate the request
// after either method returns.
// Prepare, PrepareExecution, and Run copy the request's maps, pointers, and
// nested JSON-compatible values before using them. The caller may mutate the
// request after any method returns. A successful PrepareExecution retains its
// own private execution snapshot for RunPrepared.
type RunRequest struct {
// PromptID is the required non-empty prompt identifier.
PromptID string
@@ -98,12 +100,14 @@ type RunRequest struct {
// opaque consumer metadata, not a credential, and may be exposed in
// prepared values, results, collaborator requests, provider requests, and
// provider observability. Callers should use stable, non-sensitive
// identifiers. An overlong direct value makes Prepare or Run return an
// error matching ErrInvalidRequest.
// identifiers. An overlong direct value makes Prepare, PrepareExecution, or
// Run return an error matching ErrInvalidRequest.
SessionID string
// APIKey is a request-scoped direct credential. It takes precedence over
// APIKeyEnv, is passed to the selected LLMClient, and is never included in
// prepared values, results, hashes, JSON, String, or GoString output.
// prepared values, results, hashes, JSON, String, or GoString output. A
// successful PrepareExecution retains it only in the opaque handle until
// RunPrepared claims the handle or Discard invalidates it.
APIKey string `json:"-"`
// Inputs maps prompt input names to references. A nil or empty map is valid
// only when the selected prompt and its templates require no inputs.
@@ -112,16 +116,18 @@ type RunRequest struct {
// empty maps are equivalent.
Vars map[string]string
// Execution optionally overrides individual execution settings. Nil uses
// the selected profile over its backend, when any, and framework defaults.
// the selected profile over its backend, when any, and the framework
// baseline.
Execution *ExecutionTargetOverride
// Validation optionally replaces the prompt's complete output contract. It
// does not merge individual fields. Nil uses the prompt contract.
Validation *OutputContract
}
// PreparedRun contains prepared prompt execution state. It does not include
// resolved API key values, model output, validation results, or internal target
// presence metadata. PreparedRun has a stable JSON representation.
// PreparedRun contains prepared prompt execution state returned by
// [Engine.Prepare] or [PreparedExecution.Details]. It does not include resolved
// API key values, model output, validation results, or internal target presence
// metadata. PreparedRun has a stable JSON representation.
//
// All maps, slices, pointers, and schema values are caller-owned copies. JSON
// timestamps use RFC 3339 and zero timing values are omitted. Hash formats are
@@ -139,9 +145,10 @@ type PreparedRun struct {
// SelectedBackendID equals EffectiveModelParams.BackendID. It is empty for
// an endpoint-only profile.
SelectedBackendID string `json:"selected_backend_id,omitempty"`
// EffectiveModelParams contains framework defaults overlaid by the selected
// backend, profile, and then request overrides. It excludes resolved API-key
// values.
// EffectiveModelParams contains settings resolved from the framework timeout
// baseline, selected backend, profile, and then request overrides. Unset
// optional provider controls remain zero rather than reporting a provider
// default. It excludes resolved API-key values.
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
// OutputContract is the complete effective output contract.
OutputContract OutputContract `json:"output_contract"`
@@ -154,7 +161,8 @@ type PreparedRun struct {
SessionID string `json:"session_id,omitempty"`
// RenderedPromptHash is an opaque equality value for SessionID and Messages.
RenderedPromptHash string `json:"rendered_prompt_hash"`
// Messages are the rendered messages that Run passes to the LLM client.
// Messages are the rendered messages that Run or RunPrepared passes to the
// LLM client.
Messages []RenderedMessage `json:"messages"`
// StartTime is the UTC time at which preparation began.
StartTime time.Time `json:"start_time,omitempty"`
@@ -214,12 +222,15 @@ type RunResult struct {
InputHashes map[string]string `json:"input_hashes,omitempty"`
// Usage is the token accounting reported by the LLM client.
Usage TokenUsage `json:"usage"`
// StartTime is the UTC time immediately before preparation begins.
// StartTime is the UTC time immediately before ordinary Run preparation or
// after RunPrepared claims its handle.
StartTime time.Time `json:"start_time,omitempty"`
// EndTime is the UTC time after generation and validation complete.
EndTime time.Time `json:"end_time,omitempty"`
// Duration covers preparation, generation, and validation. JSON represents
// it as integer milliseconds in duration_ms and omits a zero value.
// Duration covers preparation, generation, and validation for Run. For
// RunPrepared it covers only the execution attempt after claim and excludes
// preparation and consumer-held delay. JSON represents it as integer
// milliseconds in duration_ms and omits a zero value.
Duration time.Duration `json:"-"`
}
@@ -259,10 +270,11 @@ type Artifact struct {
// ArtifactReader resolves a prompt input reference into its content.
//
// Read may be called concurrently. It must honor ctx cancellation to make
// Prepare and Run responsive to cancellation. The engine passes a copied ref
// and immediately copies the returned Artifact.Body; it does not retain either
// value. Readers supply artifact metadata, and the engine assigns an input-map
// name only when the returned artifact name is empty.
// Prepare, PrepareExecution, and Run responsive to cancellation. The engine
// passes a copied ref and immediately copies the returned Artifact.Body; it
// does not retain either value. Readers supply artifact metadata, and the
// engine assigns an input-map name only when the returned artifact name is
// empty.
//
// An injected reader owns any application-specific path containment,
// authorization, content-size, and content-type policy. It must protect
@@ -288,13 +300,17 @@ type ExecutionTarget struct {
Endpoint string `json:"endpoint"`
// Model is the provider model identifier.
Model string `json:"model"`
// Temperature is the effective sampling temperature from 0 through 2.
// Temperature is the resolved sampling temperature from 0 through 2. Zero
// leaves the field unspecified to compatible providers unless the
// corresponding ExecutionTargetPresence bit is true.
Temperature float64 `json:"temperature"`
// MaxTokens is the non-negative effective output-token limit. Zero leaves
// the limit unspecified to compatible providers unless it was an explicit
// request override.
// MaxTokens is the non-negative resolved output-token limit. Zero leaves
// the limit unspecified to compatible providers unless the corresponding
// ExecutionTargetPresence bit is true.
MaxTokens int `json:"max_tokens"`
// TopP is the effective nucleus-sampling value from 0 through 1.
// TopP is the resolved nucleus-sampling value from 0 through 1. Zero leaves
// the field unspecified to compatible providers unless the corresponding
// ExecutionTargetPresence bit is true.
TopP float64 `json:"top_p"`
// TimeoutSeconds is the non-negative per-generation deadline. Zero disables
// this deadline without disabling caller cancellation or the transport cap.
@@ -310,6 +326,63 @@ type ExecutionTarget struct {
ExtraParams map[string]any `json:"extra_params"`
}
// ProfileInspection is the caller-owned result of [Engine.InspectProfile].
// It has no stable JSON representation.
//
// EffectiveModelParams contains a copied effective target. APIKeyRequired is
// separate from that target to preserve ExecutionTarget's general execution
// and stable JSON contracts.
type ProfileInspection struct {
// ProfileID is the trimmed, exact profile ID inspected by the engine.
ProfileID string
// EffectiveModelParams contains settings resolved from the framework timeout
// baseline, selected backend, and then profile, without a request override.
// Unset optional provider controls remain zero rather than reporting a
// provider default. APIKeyEnv is an environment-variable name, never its
// credential value.
EffectiveModelParams ExecutionTarget
// APIKeyRequired reports that a later execution must supply a direct API
// key or an explicit request environment override. It is mutually exclusive
// with a nonblank EffectiveModelParams.APIKeyEnv.
APIKeyRequired bool
}
// PromptInputDefinition describes one declared prompt input.
// It has no stable JSON representation.
type PromptInputDefinition struct {
// Name is the normalized prompt input name.
Name string
// Required is the prompt definition's declared required flag. When true,
// preparation fails if the input is omitted. A false value does not account
// for input references in message or session-ID templates.
Required bool
// ContentType is the declared input media-type metadata.
ContentType string
// Description is the declared human-readable input description.
Description string
}
// PromptInspection is the caller-owned result of [Engine.InspectPrompt].
// It has no stable JSON representation.
//
// Inputs contains copied declared input metadata in definition order.
// OutputContract is the normalized contract declared by the prompt definition,
// rather than a request-level effective override. PromptHash is opaque.
type PromptInspection struct {
// PromptID is the normalized ID of the selected prompt definition.
PromptID string
// PromptVersion is the normalized version of the selected prompt definition.
PromptVersion string
// PromptHash is the opaque equality value for the selected definition.
PromptHash string
// DefaultProfileID is declared metadata and is not resolved by inspection.
DefaultProfileID string
// Inputs contains caller-owned declared input metadata in definition order.
Inputs []PromptInputDefinition
// OutputContract is the normalized contract declared by the definition.
OutputContract OutputContract
}
// ExecutionTargetOverride represents per-request runtime setting overrides and
// has no stable JSON representation.
//
@@ -317,19 +390,26 @@ type ExecutionTarget struct {
// fields replace profile values and preserve explicit zero or empty values. A
// non-empty ExtraParams map replaces the complete profile or backend map
// rather than merging keys. Empty string fields, nil pointers, and a nil or
// empty ExtraParams map inherit the selected profile over its backend, when
// any, and framework defaults.
// empty ExtraParams map inherit lower-precedence values. An optional provider
// control that remains zero is unspecified; TimeoutSeconds retains its
// framework deadline when no higher-precedence value is present.
type ExecutionTargetOverride struct {
// Endpoint replaces the profile or backend endpoint when non-empty without
// changing the effective BackendID.
Endpoint string
// Model replaces the profile model when non-empty.
Model string
// Temperature, when non-nil, must point to a value from 0 through 2.
// Temperature, when non-nil, must point to a value from 0 through 2. A
// pointed-to zero is explicitly present; nil inherits a lower-precedence
// value and otherwise leaves the provider control unspecified.
Temperature *float64
// MaxTokens, when non-nil, must point to a non-negative value.
// MaxTokens, when non-nil, must point to a non-negative value. A pointed-to
// zero is explicitly present; nil inherits a lower-precedence value and
// otherwise leaves the provider control unspecified.
MaxTokens *int
// TopP, when non-nil, must point to a value from 0 through 1.
// TopP, when non-nil, must point to a value from 0 through 1. A pointed-to
// zero is explicitly present; nil inherits a lower-precedence value and
// otherwise leaves the provider control unspecified.
TopP *float64
// TimeoutSeconds, when non-nil, must point to a non-negative value. A
// pointed-to zero disables the per-generation deadline.
@@ -360,9 +440,12 @@ type ExecutionTargetOverride struct {
// use profile YAML api_key_env with file and FS profile sources. Profile has no
// stable JSON representation.
//
// WithProfiles validates and copies Profile values during NewEngine. Numeric
// zero, blank strings, and an empty ExtraParams map inherit framework defaults;
// use ExecutionTargetOverride pointer fields to request explicit numeric zero.
// WithProfiles validates and copies Profile values during NewEngine. Zero
// Temperature, MaxTokens, and TopP values and blank ServiceTier and
// ReasoningEffort values leave those provider controls unspecified. A zero
// TimeoutSeconds retains the framework deadline, while an empty ExtraParams map
// inherits backend request defaults. Use ExecutionTargetOverride pointer fields
// to request an explicit numeric zero.
type Profile struct {
// ID is the required non-blank profile identifier. WithProfiles trims it.
ID string
@@ -376,19 +459,19 @@ type Profile struct {
Endpoint string
// Model is the required non-blank provider model identifier.
Model string
// Temperature is from 0 through 2. Zero inherits the framework default.
// Temperature is from 0 through 2. Zero leaves the provider control
// unspecified.
Temperature float64
// MaxTokens is non-negative. Zero inherits the framework default.
// MaxTokens is non-negative. Zero leaves the provider control unspecified.
MaxTokens int
// TopP is from 0 through 1. Zero inherits the framework default rather than
// selecting an explicit zero.
// TopP is from 0 through 1. Zero leaves the provider control unspecified
// rather than selecting an explicit zero.
TopP float64
// TimeoutSeconds is non-negative. Zero inherits the framework default.
// TimeoutSeconds is non-negative. Zero retains the framework deadline.
TimeoutSeconds int
// ServiceTier is optional; a blank value inherits the framework default.
// ServiceTier is optional; a blank value leaves it unspecified.
ServiceTier string
// ReasoningEffort is optional; a blank value inherits the framework
// default.
// ReasoningEffort is optional; a blank value leaves it unspecified.
ReasoningEffort string
// APIKeyRequired clears a backend's inherited API-key environment name and
// requires a non-blank RunRequest.APIKey unless the request explicitly
@@ -559,25 +642,26 @@ type StructuredOutputJSONSpec struct {
Schema any `json:"schema"`
}
// LLMClient executes rendered prompts for [Engine.Run].
// LLMClient executes rendered prompts for [Engine.Run] and
// [Engine.RunPrepared].
//
// Generate is scheduled according to the resolved backend's capacity policy.
// It may still be called concurrently for different backend pools or unlimited
// backends. Cancellation while waiting for capacity can prevent Generate from
// being called. Once invoked, it must honor context cancellation to make Run
// responsive to cancellation. The request and all nested maps, slices, and
// pointers are client-owned copies and may be mutated or retained without
// affecting engine state.
// and RunPrepared responsive to cancellation. The request and all nested maps,
// slices, and pointers are client-owned copies and may be mutated or retained
// without affecting engine state.
//
// Generate receives rendered messages and may receive a direct API key. A
// client must protect those values and any raw output in its logging, storage,
// and retained copies. It is responsible for the cancellation behavior of any
// work it starts and for synchronizing access to retained or shared data.
//
// A returned error makes Run return ErrLLMGenerate while preserving the client
// error through errors.Is. A nil response with a nil error also produces
// ErrLLMGenerate. Promptkit copies the non-nil response before returning from
// Run.
// A returned error makes Run or RunPrepared return ErrLLMGenerate while
// preserving the client error through errors.Is. A nil response with a nil
// error also produces ErrLLMGenerate. Promptkit copies the non-nil response
// before returning from either method.
type LLMClient interface {
Generate(context.Context, GenerateRequest) (*GenerateResponse, error)
}