18 Commits

Author SHA1 Message Date
31f2ce3a09 Document Promptkit v0.5.0 2026-08-01 13:18:46 +00:00
fd06e4ca6b Clean up roadmap and fallback profile guidance 2026-08-01 13:16:26 +00:00
e63b8de1e9 Complete application fallback profile implementation 2026-08-01 12:39:01 +00:00
9354d2b373 Add application fallback profile sources 2026-08-01 12:37:01 +00:00
01ca5430bd Move profile composition to the engine facade 2026-08-01 12:31:50 +00:00
ae2179d103 Complete optional parameter omission 2026-08-01 02:43:02 +00:00
a248433d0f Omit unset optional request parameters 2026-08-01 02:41:44 +00:00
bd6cffc9d0 Prepare documentation for Promptkit v0.4.0 2026-07-30 23:48:25 +00:00
e40c4f182b Document structured capacity errors 2026-07-30 23:26:44 +00:00
7428e50c2c Expose structured capacity errors 2026-07-30 23:24:04 +00:00
63c67a4520 Add internal capacity error identity 2026-07-30 23:21:27 +00:00
25a7052a3d Clarify prompt input requirement documentation 2026-07-30 22:10:33 +00:00
fc3255967e Document prompt inspection API 2026-07-30 21:06:33 +00:00
e920168b30 Expose prompt inspection through the engine 2026-07-30 21:03:47 +00:00
272b6a4bc1 Add internal prompt inspection 2026-07-30 21:00:05 +00:00
dde48a31fc Document profile inspection API 2026-07-30 19:57:07 +00:00
242eace4a7 Expose profile inspection through the engine 2026-07-30 19:52:00 +00:00
0bf5f88136 Add internal profile inspection resolution 2026-07-30 19:48:09 +00:00
43 changed files with 2307 additions and 2848 deletions

View File

@@ -33,6 +33,12 @@ boundary and constraints that framework work must preserve.
## Release Guidance
Consumers upgrading from `v0.4.0` to `v0.5.0` should read the
[v0.5.0 changelog and migration guide](docs/releases/v0.5.0.md).
Consumers upgrading from `v0.3.0` to `v0.4.0` should read the
[v0.4.0 changelog and adoption guide](docs/releases/v0.4.0.md).
Consumers upgrading from `v0.2.0` to `v0.3.0` should read the
[v0.3.0 changelog](docs/releases/v0.3.0.md).

View File

@@ -60,7 +60,18 @@ func TestEngineRejectsRunBeforeCompletionWhenAdmissionIsFull(t *testing.T) {
awaitCapacitySignal(t, reader.entered, "first artifact read")
result, err := engine.Run(context.Background(), capacityInputRequest("http://second.example/v1"))
canceledContext, cancel := context.WithCancel(context.Background())
cancel()
result, err := engine.Run(canceledContext, capacityInputRequest("http://canceled.example/v1"))
if result != nil || !errors.Is(err, context.Canceled) {
t.Fatalf("canceled capacity admission=(%+v, %v), want context cancellation", result, err)
}
var canceledCapacityErr *promptkit.CapacityError
if errors.Is(err, promptkit.ErrCapacityExceeded) || errors.As(err, &canceledCapacityErr) {
t.Fatalf("canceled admission exposed capacity rejection: %v", err)
}
result, err = engine.Run(context.Background(), capacityInputRequest("http://second.example/v1"))
if result != nil {
t.Fatalf("capacity rejection returned partial result: %+v", result)
}
@@ -70,6 +81,21 @@ func TestEngineRejectsRunBeforeCompletionWhenAdmissionIsFull(t *testing.T) {
if errors.Is(err, promptkit.ErrInvalidRequest) || errors.Is(err, promptkit.ErrLLMGenerate) {
t.Fatalf("capacity rejection had an unrelated category: %v", err)
}
var capacityErr *promptkit.CapacityError
if !errors.As(err, &capacityErr) || capacityErr == nil {
t.Fatalf("capacity rejection=%v, want CapacityError", err)
}
if capacityErr.BackendID != "limited" {
t.Fatalf("capacity backend ID=%q, want limited", capacityErr.BackendID)
}
capacityErr.BackendID = "changed"
result, err = engine.Run(context.Background(), capacityInputRequest("http://third.example/v1"))
var subsequentCapacityErr *promptkit.CapacityError
if result != nil || !errors.As(err, &subsequentCapacityErr) ||
subsequentCapacityErr == nil || subsequentCapacityErr.BackendID != "limited" {
t.Fatalf("subsequent capacity rejection=(%+v, %v), want independent limited CapacityError", result, err)
}
if calls := reader.callCount(); calls != 1 {
t.Fatalf("artifact calls=%d, want only the admitted run", calls)
}
@@ -174,6 +200,19 @@ func TestCapacityExceededSentinelContract(t *testing.T) {
if promptkit.ErrCapacityExceeded == nil {
t.Fatal("ErrCapacityExceeded is nil")
}
var nilCapacityErr *promptkit.CapacityError
zeroCapacityErr := &promptkit.CapacityError{}
populatedCapacityErr := &promptkit.CapacityError{BackendID: "limited"}
for _, capacityErr := range []error{nilCapacityErr, zeroCapacityErr} {
if !errors.Is(capacityErr, promptkit.ErrCapacityExceeded) {
t.Fatalf("capacity error=%v, want ErrCapacityExceeded", capacityErr)
}
}
var discoveredCapacityErr *promptkit.CapacityError
if !errors.As(populatedCapacityErr, &discoveredCapacityErr) || discoveredCapacityErr != populatedCapacityErr {
t.Fatalf("populated capacity error is not discoverable: %v", populatedCapacityErr)
}
for _, unrelated := range []error{
promptkit.ErrInvalidConfig,
promptkit.ErrInvalidRequest,
@@ -181,7 +220,8 @@ func TestCapacityExceededSentinelContract(t *testing.T) {
promptkit.ErrValidation,
} {
if errors.Is(promptkit.ErrCapacityExceeded, unrelated) ||
errors.Is(unrelated, promptkit.ErrCapacityExceeded) {
errors.Is(unrelated, promptkit.ErrCapacityExceeded) ||
errors.Is(populatedCapacityErr, unrelated) {
t.Fatalf("ErrCapacityExceeded aliases unrelated sentinel %v", unrelated)
}
}

40
capacity_error.go Normal file
View File

@@ -0,0 +1,40 @@
package promptkit
import (
"fmt"
"strings"
)
// CapacityError reports bounded admission rejected for a selected backend.
//
// Engine-produced values identify only rejection at Promptkit's bounded
// [Engine.Run] or [Engine.RunPrepared] admission boundary. BackendID is the
// normalized registered backend ID used for routing and capacity; endpoint
// overrides do not change it. Every engine-produced value is nonnil and has a
// nonblank BackendID. Provider errors, active-generation waiting, and caller
// cancellation are not represented by this type.
//
// Callers own returned values and may mutate BackendID without affecting engine
// state or another error. CapacityError and its default Go encoding have no
// stable JSON contract. Consumer-constructed values do not establish that an
// engine rejected work.
type CapacityError struct {
// BackendID is the normalized registered backend ID whose admission was
// rejected.
BackendID string
}
// Error returns diagnostic wording that is not a parsing contract. It is safe
// to call on a nil receiver or a value with a blank BackendID.
func (e *CapacityError) Error() string {
if e == nil || strings.TrimSpace(e.BackendID) == "" {
return ErrCapacityExceeded.Error()
}
return fmt.Sprintf("backend %q admission: %v", e.BackendID, ErrCapacityExceeded)
}
// Unwrap returns ErrCapacityExceeded so errors.Is and errors.As can be used
// together. It is safe to call on a nil receiver or a zero value.
func (e *CapacityError) Unwrap() error {
return ErrCapacityExceeded
}

View File

@@ -170,6 +170,40 @@ func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
}
}
func fromDomainProfileInspection(inspection *domain.ProfileInspection) *ProfileInspection {
if inspection == nil {
return nil
}
return &ProfileInspection{
ProfileID: inspection.ProfileID,
EffectiveModelParams: fromDomainExecutionTarget(inspection.EffectiveModelParams),
APIKeyRequired: inspection.APIKeyRequired,
}
}
func fromDomainPromptInspection(inspection *domain.PromptInspection) *PromptInspection {
if inspection == nil {
return nil
}
inputs := make([]PromptInputDefinition, len(inspection.Inputs))
for i, input := range inspection.Inputs {
inputs[i] = PromptInputDefinition{
Name: input.Name,
Required: input.Required,
ContentType: input.ContentType,
Description: input.Description,
}
}
return &PromptInspection{
PromptID: inspection.PromptID,
PromptVersion: inspection.PromptVersion,
PromptHash: inspection.PromptHash,
DefaultProfileID: inspection.DefaultProfileID,
Inputs: inputs,
OutputContract: fromDomainOutputContract(inspection.OutputContract),
}
}
func fromDomainExecutionTargetPresence(presence domain.ExecutionTargetPresence) ExecutionTargetPresence {
return ExecutionTargetPresence{
Temperature: presence.Temperature,

40
doc.go
View File

@@ -3,25 +3,28 @@
//
// Applications construct an [Engine] with [NewEngine], select filesystem or
// in-memory sources and optional engine-scoped [Backend] registrations, and
// call [Engine.Prepare], [Engine.PrepareExecution], [Engine.Run], or
// [Engine.RunPrepared]. Concrete registries, repositories, validators, and the
// built-in OpenAI-compatible client remain internal implementation details.
// call [Engine.InspectPrompt], [Engine.InspectProfile], [Engine.Prepare],
// [Engine.PrepareExecution], [Engine.Run], or [Engine.RunPrepared]. Concrete
// registries, repositories, validators, and the built-in OpenAI-compatible
// client remain internal implementation details.
//
// # Concurrency and ownership
//
// An Engine supports concurrent Prepare, PrepareExecution, Run, and RunPrepared
// calls. Engine-local backend policies bound admitted Run and RunPrepared calls
// and model generations where configured, while different backend pools and
// unlimited backends continue independently. An injected [LLMClient] or
// [ArtifactReader] can therefore still receive concurrent calls and must be
// safe for that use.
// An Engine supports concurrent InspectPrompt, InspectProfile, Prepare,
// PrepareExecution, Run, and RunPrepared calls. Engine-local backend policies
// bound admitted Run and RunPrepared calls and model generations where
// configured, while different backend pools and unlimited backends continue
// independently. An injected [LLMClient] or [ArtifactReader] can therefore
// still receive concurrent calls and must be safe for that use.
//
// NewEngine copies in-memory profiles and backend definitions. Prepare,
// PrepareExecution, and Run copy request maps, slices, pointer values, and
// JSON-compatible extra parameters before using them. Returned values and
// values passed to extension interfaces are likewise isolated from engine
// state. Callers own those copies and may mutate them after the call that
// supplied or returned them.
// JSON-compatible extra parameters before using them. InspectPrompt and
// InspectProfile return copied inspection values. Returned values and values
// passed to extension interfaces are likewise isolated from engine state.
// Callers own those copies and may mutate them after the call that supplied or
// returned them. Returned structured errors are likewise caller-owned and may
// be mutated without affecting engine state or another error.
//
// # Security and sensitive data
//
@@ -47,11 +50,12 @@
// [GenerateResponse], [ExecutionTargetPresence], and the string value types
// used by those values.
//
// Construction and handle values, including [Config], [Backend], [RunRequest],
// [ArtifactRef], [ExecutionTargetOverride], [Profile],
// [OpenAICompatibleProfileConfig], and [PreparedExecution], do not have stable
// JSON representations. Direct API keys are nevertheless excluded from JSON
// for every public value.
// Construction, inspection, handle, and error values, including [Config],
// [Backend], [RunRequest], [ArtifactRef], [ExecutionTargetOverride], [Profile],
// [OpenAICompatibleProfileConfig], [ProfileInspection],
// [PromptInputDefinition], [PromptInspection], [PreparedExecution], and
// [CapacityError], do not have stable JSON representations. Direct API keys
// are nevertheless excluded from JSON for every public value.
//
// JSON timestamps use time.Time's RFC 3339 encoding and are omitted when zero.
// PreparedRun and RunResult durations are encoded as integer milliseconds in

View File

@@ -40,6 +40,60 @@ validation, and default transport behavior. Source discovery, format
validation, and profile precedence are defined by the
[framework format reference](../formats.md).
## Supply Embedded Application Defaults
Use `WithFallbackProfileFS` when an application packages profile definitions
that should apply unless an operator provides an ordinary configured profile
with the same ID. For example, an application can embed its defaults while
continuing to use `ProfileDir` for operator overrides:
```go
//go:embed profiles/*.yaml
var applicationProfiles embed.FS
engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: "prompts",
ProfileDir: operatorProfileDir,
},
promptkit.WithFallbackProfileFS(applicationProfiles, "profiles"),
)
```
Keep application-owned profile IDs and definitions in the embedded source.
Use the ordinary configured profile source for operator overrides. Leave
`operatorProfileDir` empty when the operator did not configure an override
directory; a non-empty path names an authoritative higher-precedence source,
so an unavailable or unreadable directory is an error rather than a reason to
fall back. The
[framework format reference](../formats.md#source-and-profile-precedence)
owns the exact profile format and lookup order; the
[`WithFallbackProfileFS` GoDoc](../../engine.go) owns its option contract and
validation rules.
## Inspect A Prompt Before Preparation
Use [`Engine.InspectPrompt`](../../engine.go) to check one configured prompt's
declared inputs and output workflow without creating placeholder inputs or
resolving a profile:
```go
inspection, err := engine.InspectPrompt(ctx, "meeting.summary", "")
if err != nil {
return err
}
for _, input := range inspection.Inputs {
// Compare the declared input with application configuration.
}
```
Use this configuration-time boundary when the application needs only the
declared prompt interface. Use `InspectProfile` separately when it must also
check a configured profile. Use `Prepare` when it needs inputs, schemas, or
rendered messages, and use prepared execution when that work must remain tied
to later execution. The method's [GoDoc](../../engine.go) owns exact fields,
hash, ownership, and error semantics.
## Prepare Without Model Execution
[`Engine.Prepare`](../../engine.go) resolves the selected prompt and profile,
@@ -128,13 +182,41 @@ replace execution settings or the complete output contract.
The [public value GoDoc](../../types.go) defines nil, empty, zero, replacement,
copy, and credential behavior. The
[framework format reference](../formats.md) defines how those request values
interact with prompt definitions, file-backed profiles, built-ins, schemas,
and framework defaults.
interact with prompt definitions, file-backed and application fallback
profiles, built-ins, schemas, and framework defaults.
For programmatic profiles,
[`OpenAICompatibleProfile`](../../profiles.go) converts ordinary
OpenAI-compatible settings into a value accepted by `WithProfiles`.
### Inspect A Profile Before Prompt Work
Use [`Engine.InspectProfile`](../../engine.go) to validate one configured
profile without constructing a synthetic prompt or placeholder inputs. It
resolves the profile's effective target but does not prepare or execute a
prompt:
```go
inspection, err := engine.InspectProfile(ctx, profileID)
if err != nil {
return err
}
target := inspection.EffectiveModelParams
if target.APIKeyEnv != "" {
// Apply application policy for the named environment variable.
} else if inspection.APIKeyRequired {
// Arrange a direct credential before later execution.
}
```
Use this configuration-time boundary when only the profile and its target need
checking. Use `Prepare` when the application also needs prompt, input, schema,
or rendering work; use prepared execution when that work must remain tied to a
later execution. Inspection reports credential requirements but leaves the
timing of credential enforcement to the application. The method's
[GoDoc](../../engine.go) owns its exact result and error contract.
### Set A Per-Run Session And Reasoning
Supply a direct session ID when one prompt should be correlated with a
@@ -325,6 +407,11 @@ When a limited backend has admitted all active and waiting calls, handle
```go
result, err := engine.Run(ctx, request)
if errors.Is(err, promptkit.ErrCapacityExceeded) {
var capacityErr *promptkit.CapacityError
if errors.As(err, &capacityErr) {
// Record capacityErr.BackendID using application-owned diagnostics.
}
// Apply application policy: shed work, report overload, or retry later.
}
```
@@ -332,8 +419,9 @@ if errors.Is(err, promptkit.ErrCapacityExceeded) {
A rejected call returns no partial result and does not invoke the model
client. Promptkit does not prescribe retries or map this error to an HTTP
status; those choices remain with the consuming application. The
[`Engine.Run` and error GoDoc](../../engine.go) owns exact error and
cancellation identities.
[`CapacityError` GoDoc](../../capacity_error.go) owns the exact typed-error
contract, while the [`Engine.Run` and error GoDoc](../../engine.go) owns broad
error and cancellation identities.
## Application Boundary

View File

@@ -44,98 +44,3 @@ Start with:
For cross-cutting changes, follow every applicable row. Do not create
placeholder documents for packages, APIs, or integrations that do not yet
exist.
## Maintainer-Run Validation
Promptkit does not currently use hosted CI. Maintainers are responsible for
running the documented checks before accepting changes. Run the default Go
validation from the Promptkit repository root:
```sh
go test ./...
go test -race ./...
go vet ./...
go build ./...
go run ./examples/go-library/prepare
```
Check formatting across every tracked Go file:
```sh
gofmt -l $(git ls-files '*.go')
```
The formatting command must produce no paths. Follow every added or changed
Markdown link and confirm its target exists. Finally, check whitespace:
```sh
git diff --check
```
Documentation-only work does not require unrelated new tests, but it still
requires link validation and `git diff --check`. Run the Go validation whenever
documentation changes commands, examples, generated output, or another
behavior checked by the module.
## Focused Validation
Use focused checks while iterating, then run the complete validation sequence
before accepting the change. The root package supports:
```sh
go test .
go vet .
go build .
```
Filter tests by name without assuming a fixed internal package layout:
```sh
go test ./... -run 'TestName'
```
Replace `TestName` with a useful regular expression. Target only paths that
exist, and consult the internal component overview for their owning
documentation. A filtered or package-specific run does not replace the
complete repository validation.
## Coordinated Work With Scriptorium
Promptkit and Scriptorium must remain independently valid. For temporary local
integration, use either a Go workspace outside both repositories or an
uncommitted replacement in the consuming module.
If the repositories are sibling directories, run the workspace commands from
their parent directory:
```sh
go work init ./promptkit ./scriptorium
go work sync
```
Use the workspace only for coordinated local checks. From the same parent
directory, remove it when finished:
```sh
rm -f go.work go.work.sum
```
Alternatively, from the Scriptorium repository root, temporarily point its
Promptkit dependency at the sibling checkout:
```sh
go mod edit -replace gitea.maximumdirect.net/eric/promptkit=../promptkit
```
After coordinated checks, remove the replacement and reconcile module
metadata:
```sh
go mod edit -dropreplace gitea.maximumdirect.net/eric/promptkit
go mod tidy
```
Never commit `go.work`, `go.work.sum`, or a local filesystem `replace`
directive. Before committing in either repository, inspect its module files and
working tree independently. Published consumer versions must depend on a tagged
Promptkit version, not a workspace, local replacement, or unpublished commit.

View File

@@ -58,6 +58,11 @@ When a request omits a version, the selected prompt ID must identify exactly
one definition. When it supplies a version, the ID and version pair must be
unique.
Exact prompt inspection uses this same configured source, strict decoding,
referenced content-file resolution, and ID/version selection. It reports the
selected definition's declared metadata without changing the prompt format or
executing the definition.
### Inputs
Each `inputs` item has these fields:
@@ -152,7 +157,7 @@ extra_params:
| Field | Required | Meaning |
| --- | --- | --- |
| `id` | yes | Non-empty profile identifier. IDs must be unique within one source. |
| `backend` | unless `endpoint` is present | Backend registry ID. It is trimmed and registry membership is checked when the profile is prepared. |
| `backend` | unless `endpoint` is present | Backend registry ID. It is trimmed and registry membership is checked when the profile is prepared or inspected. |
| `endpoint` | unless `backend` is present | Non-empty OpenAI-compatible base URL, including an API version path when required. When both connection fields are present, this overrides the backend endpoint without changing backend identity. |
| `model` | yes | Non-empty provider model name. |
| `temperature` | no | Number from 0 through 2. |
@@ -183,25 +188,27 @@ they also cannot collide with the standard fields listed in the
Execution settings resolve in this order:
1. framework defaults;
1. the framework timeout baseline;
2. the selected backend, when the profile names one;
3. the selected profile; and
4. request `ExecutionTargetOverride` values.
The framework defaults are:
The framework baseline is:
| Setting | Default |
| --- | --- |
| `temperature` | `0` |
| `max_tokens` | `0` |
| `top_p` | `1` |
| `temperature` | Unspecified and omitted from compatible provider requests unless a profile or runtime override selects it. |
| `max_tokens` | Unspecified and omitted from compatible provider requests unless a profile or runtime override selects it. |
| `top_p` | Unspecified and omitted from compatible provider requests unless a profile or runtime override selects it. |
| `timeout_seconds` | `600` |
Numeric zero in a file or in-memory profile means that the profile does not
replace the framework default. Numeric request overrides use pointers, so an
explicit zero is preserved. In particular, an explicit request
`timeout_seconds` of zero disables the per-generation deadline while leaving
the caller context and transport timeout intact.
Numeric zero in a file or in-memory profile does not select a numeric value.
For `temperature`, `max_tokens`, and `top_p`, it leaves the provider control
unspecified. For `timeout_seconds`, it retains the framework deadline. Numeric
request overrides use pointers, so an explicit zero is retained and sent to
compatible providers. In particular, an explicit request `timeout_seconds` of
zero disables the per-generation deadline while leaving the caller context and
transport timeout intact.
Non-empty profile strings replace backend defaults, and non-empty request
strings replace both. Request reasoning is the exception: a nil
@@ -219,18 +226,25 @@ defines how the effective settings are serialized.
### Source And Profile Precedence
An explicit request profile ID takes precedence over the prompt's
`default_profile`. If neither is present, preparation fails.
`default_profile`. If neither is present, preparation fails. Exact profile
inspection instead takes one explicit profile ID and does not use a prompt
default.
Profile sources resolve matching IDs in this order:
1. in-memory profiles supplied with `WithProfiles`;
2. a profile file, `fs.FS`, or configured profile directory; and
3. embedded built-in profiles.
2. the ordinary configured source selected by a profile file, `fs.FS`, or
configured profile directory;
3. application fallback profiles supplied with `WithFallbackProfileFS`; and
4. embedded built-in profiles.
A higher-precedence source falls back only when the profile is absent. An
invalid matching profile is an error and does not fall back. In-memory
`Profile` values follow the same ranges as YAML profiles. They use
`APIKeyRequired` for request-scoped credentials instead of `api_key_env`.
A profile source supplies a complete definition; definitions and their fields
are not merged across sources. A higher-precedence source falls back only when
the requested profile ID is absent. An invalid matching profile is an error and
does not fall back. In-memory `Profile` values follow the same ranges as YAML
profiles. They use `APIKeyRequired` for request-scoped credentials instead of
`api_key_env`. Preparation and exact profile inspection use this same source
precedence.
## Built-In Profile Catalog
@@ -238,8 +252,8 @@ Every built-in selects the `openrouter` backend. The engine's built-in backend
registry supplies `https://openrouter.ai/api/v1` and the environment-variable
name `OPENROUTER_API_KEY`, so individual profiles contain only model and
generation settings. Built-in profile files do not repeat those connection
values. A custom or in-memory profile with the same profile ID takes
precedence.
values. A configured, application fallback, or in-memory profile with the same
profile ID takes precedence.
| Provider | ID | Model |
| --- | --- | --- |

View File

@@ -53,8 +53,9 @@ never also sent as a session header.
The client conditionally includes:
- `temperature`, `max_tokens`, and `top_p` when non-zero or explicitly
present;
- `temperature`, `max_tokens`, and `top_p` only when selected by a profile or
runtime override, including an explicit runtime zero; they are absent when
unspecified;
- non-empty `service_tier` and effective `reasoning_effort`; an explicitly
disabled reasoning setting is empty and therefore omitted; and
- `response_format` for JSON Schema structured output, including its name,

View File

@@ -38,10 +38,10 @@ output contract, but before schema loading, artifact loading, or rendering.
rechecks credential availability, and then asks the manager to admit the
frozen backend before generation.
Admission is immediate: a limited pool either reserves a slot or returns the
internal `ErrCapacityExceeded` identity. The root facade maps that identity to
the public error without treating it as an invalid request or generation
failure.
Admission is immediate: a limited pool either reserves a slot or returns only
the internal `ErrCapacityExceeded` identity. The runner attaches the selected
backend identity at its use-case boundary, and the root facade translates that
typed value without treating it as an invalid request or generation failure.
The total admitted bound is the active-generation limit plus its configured
waiting capacity. The returned release function is idempotent. The runner

View File

@@ -11,7 +11,7 @@ contributor workflow and validation.
| Component | Implemented responsibility | References |
| --- | --- | --- |
| Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request and result values, opaque prepared-execution handles, profile construction, extension interfaces, value conversion, redacted formatting, public error mapping, and engine-local assembly. | [Package GoDoc](../../doc.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine assembly](../../engine.go) |
| Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request, result, prompt-inspection, and profile-inspection values, opaque prepared-execution handles, profile construction, extension interfaces, value conversion, redacted formatting, typed capacity errors, public error mapping, and engine-local profile-source assembly including application fallbacks. | [Package GoDoc](../../doc.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine assembly](../../engine.go) |
| `examples/go-library/prepare` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, and `Prepare`. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
| `internal/backend` | Constructs each engine's immutable registry from the built-in OpenRouter definition and consumer additions, validates and defensively copies definitions through the shared JSON-value package, and consumes the LLM-owned OpenAI-compatible reserved request-field rule. | [Backend registry](../../internal/backend/registry.go) |
@@ -22,12 +22,12 @@ contributor workflow and validation.
| `internal/jsonvalue` | Validates and deeply copies JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
| `internal/promptdef` | Loads strictly decoded, validated prompt definitions from filesystem and `fs.FS` sources, including version selection and contained file-backed message content. | [Framework formats](../formats.md), [prompt-definition repository](../../internal/promptdef/filesystem_repository.go) |
| `internal/profile` | Loads strictly decoded, validated execution profiles, including backend selection, from filesystem and `fs.FS` sources and composes repositories with error-preserving fallback. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go) |
| `internal/profile/builtin` | Embeds the built-in profile catalog, whose entries select OpenRouter, and combines it with an optional primary repository. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) |
| `internal/profile/builtin` | Embeds the built-in profile catalog, whose entries select OpenRouter. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) |
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates frozen validation plans for prepared execution. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
| `internal/usecase` | Resolves backend, profile, and request settings and coordinates preparation, ordinary execution, and one-attempt prepared execution across internal sources, rendering, artifact loading, generation, validation, capacity, and optional repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
| `internal/usecase` | Resolves prompt definitions and hashes, profiles, backends, and targets for exact inspection and request settings for preparation, and coordinates ordinary execution and one-attempt prepared execution across internal sources, rendering, artifact loading, generation, validation, capacity, and optional repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
The root package assembles these internal components without exposing their
representations. Consumers depend only on the root facade.

View File

@@ -28,6 +28,34 @@ constructor does not enable one.
Each invocation carries its state in request, prepared-run, and result values.
The runner has no durable run or session store.
## Shared Prompt Selection
The runner uses one prompt-selection and hashing boundary for ordinary
preparation and exact prompt inspection. Preparation retains its early
request-ID check before direct-session normalization; both operations then use
the configured prompt repository to select one definition, load referenced
message content, and calculate the same prompt hash.
Inspection stops after that structural lookup. It does not parse templates or
touch profile, artifact, schema, renderer, validator, admission, or model
collaborators. The root [`Engine.InspectPrompt`](../../engine.go) GoDoc owns
the public operation's exact contract.
## Shared Profile Selection
The runner uses one profile-selection and target-resolution boundary for
ordinary preparation and exact profile inspection. Preparation first selects a
request profile or a prompt default; inspection begins with its required
explicit profile ID. Both then apply the ordinary source precedence, resolve a
named backend, and construct the effective target from framework, backend, and
profile values.
Inspection stops after the resulting endpoint and model are structurally
validated. It does not check credential availability or perform prompt,
artifact, schema, rendering, admission, or model-client work. The root
[`Engine.InspectProfile`](../../engine.go) GoDoc owns the public operation's
exact contract.
## Shared Preparation Pipeline
`Prepare` and `Run` share one private preparation pipeline split at the point
@@ -124,15 +152,17 @@ validation failures. Wrapping preserves the package identities mapped by the
public facade and retains collaborator identities where they are part of the
internal contract.
Admission capacity exhaustion retains the internal capacity identity and adds
the selected backend ID as context. It is not recategorized as an invalid
request or generation failure, and no partial result is returned. A context
already done at admission retains its context identity directly. Cancellation
while waiting for an active generation permit prevents client invocation when
it wins the grant race; the model-client boundary then preserves the context
error through the generation-failure category. Deferred release restores the
admission lease on preparation, generation, validation, repair, and
cancellation failures.
Admission capacity exhaustion retains the internal capacity identity. At the
use-case boundary, the runner attaches the selected backend ID in an internal
typed error, and the root facade copies that value into the public
[`CapacityError`](../../capacity_error.go) without parsing diagnostic text. It
is not recategorized as an invalid request or generation failure, and no
partial result is returned. A context already done at admission retains its
context identity directly. Cancellation while waiting for an active generation
permit prevents client invocation when it wins the grant race; the model-client
boundary then preserves the context error through the generation-failure
category. Deferred release restores the admission lease on preparation,
generation, validation, repair, and cancellation failures.
Other context cancellation propagates through the invoked collaborator and is
classified by the owning operation.

View File

@@ -16,6 +16,11 @@ validation modes, built-in catalog, and source precedence.
definitions, selects an ID and optional version, and resolves file-backed
message content within the selected operating-system or `fs.FS` source.
Exact prompt inspection performs one point-in-time lookup through that same
repository and validates referenced message content before returning declared
metadata. It does not parse templates or read profile, input, or schema
sources, and it does not retain the definition for a later execution.
Its package tests own prompt selection, strict decoding, definition validation,
duplicate detection, and source containment:
[prompt-definition repository tests](../../internal/promptdef/repository_test.go).
@@ -23,21 +28,30 @@ duplicate detection, and source containment:
## Profiles And Built-Ins
`internal/profile` loads and validates execution profiles from an
operating-system filesystem or an `fs.FS`. It supports a primary repository
with fallback only when the primary reports that a profile is absent. Strict
YAML decoding recognizes the optional `backend` field, trims its value, and
requires a model plus at least one non-blank backend or endpoint. Loading does
not check registry membership because the available registry belongs to the
assembled engine; the runner checks membership during preparation.
operating-system filesystem or an `fs.FS`. Its overlay repository consults the
next repository only when the higher-precedence repository reports that a
profile is absent. Strict YAML decoding recognizes the optional `backend`
field, trims its value, and requires a model plus at least one non-blank
backend or endpoint. Loading does not check registry membership because the
available registry belongs to the assembled engine; the runner checks
membership during preparation and exact profile inspection.
`internal/profile/builtin` embeds the maintained built-in profile catalog and
can place a caller-selected repository ahead of that catalog. Every embedded
profile selects `openrouter` and inherits its endpoint and credential
environment-variable name from the built-in backend registry rather than
repeating those values. Profile behavior is owned by the
[profile repository tests](../../internal/profile/repository_test.go), while
catalog completeness, the backend-selection invariant, duplicate IDs, and
overlay behavior are owned by the
The root engine assembles profile repositories in precedence order: in-memory
profiles, one ordinary configured source, an application fallback source, then
the embedded built-in catalog. An explicit file or `fs.FS` profile source
replaces `Config.ProfileDir` within the ordinary configured-source category.
Exact profile inspection performs one point-in-time lookup through those
profile sources and checks the resolved target without reading prompt, input,
or schema sources. It does not retain that lookup for a later execution.
`internal/profile/builtin` embeds the maintained built-in profile catalog.
Every embedded profile selects `openrouter` and inherits its endpoint and
credential environment-variable name from the built-in backend registry rather
than repeating those values. Profile loading and overlay behavior are owned by
the [profile repository tests](../../internal/profile/repository_test.go),
while catalog completeness, the backend-selection invariant, and duplicate IDs
are owned by the
[built-in repository tests](../../internal/profile/builtin/repository_test.go).
## Ordinary Artifacts
@@ -79,7 +93,7 @@ captured root document.
loading and hashing, session and message rendering, and target resolution.
`RunPrepared` uses the retained source-derived state and validation plan; it
does not reopen prompt, profile, input, or schema sources and does not rerender
the request. By contrast, ordinary `Prepare` produces an inspection value only:
the request. By contrast, ordinary `Prepare` produces a preparation value only:
a later `Run` performs its own source resolution and preparation.
The [validator tests](../../internal/validate/standard_validator_test.go) own

189
docs/releases/v0.4.0.md Normal file
View File

@@ -0,0 +1,189 @@
# Promptkit v0.4.0
This supplemental changelog and adoption guide summarizes the consumer-facing
changes from `v0.3.0` to `v0.4.0`. The annotated `v0.4.0` tag is the
authoritative release record. Exact current contracts belong to the linked
GoDoc and durable documentation.
## Summary
`v0.4.0` adds four complementary capabilities:
- opaque prepared-execution handles for preparing once, inspecting safe
details, and executing the same frozen snapshot;
- exact prompt-definition inspection without profile resolution or execution;
- exact profile inspection without selecting a prompt or checking credential
availability; and
- structured backend identity on engine admission-capacity rejection.
These APIs let consumers perform more precise preflight work and retain useful
operational context without reproducing Promptkit's internal resolution logic.
## Compatibility
The release is additive for `v0.3.0` consumers. Existing uses of `Prepare`,
`Run`, backend registration, endpoint-only profiles, local-backend helpers,
runtime overrides, public JSON values, and error sentinels continue to work
without migration.
Capacity rejection now returns a structured error while continuing to match
`ErrCapacityExceeded` through `errors.Is`. Error-string wording and direct
sentinel equality were not public contracts.
The new inspection values, capacity error, and prepared-execution handle do not
have stable JSON representations. `PreparedExecution.Details` returns the
existing stable `PreparedRun` value.
## Upgrade
Update the module dependency with:
```sh
go get gitea.maximumdirect.net/eric/promptkit@v0.4.0
go mod tidy
```
Run the consuming project's ordinary and race-enabled tests after upgrading.
No source migration is required.
## Prepare Once And Execute The Same Snapshot
Consumers that need to persist preparation details before generation can now
prepare an opaque, engine-bound execution:
```go
prepared, err := engine.PrepareExecution(ctx, request)
if err != nil {
// Handle preparation failure.
}
defer prepared.Discard()
details := prepared.Details()
// Persist a consumer-selected, appropriately protected preparation record.
result, err := engine.RunPrepared(ctx, prepared)
```
Preparation freezes the selected sources, rendered messages, effective
settings, input content, structured-output metadata, and validation resources
needed by execution. `Details` returns a fresh, caller-owned,
credential-redacted `PreparedRun`.
A handle belongs to its creating engine and permits one execution attempt.
`RunPrepared` consumes that attempt on success and on operational failure.
`Discard` is idempotent and releases an unclaimed handle's execution-only
state. Consumers should discard handles they will not execute, particularly
when a direct request API key may be retained privately until claim or
discard.
Prepared execution does not reserve backend admission during preparation.
Credential availability and backend admission are checked when execution
begins. The execution context is independent of the preparation context.
See the
[prepared-execution consumer guide](../consumers/pkg-promptkit.md#prepare-now-and-execute-the-same-snapshot-later),
the [`PreparedExecution` GoDoc](../../prepared_execution.go), and the
[`Engine.PrepareExecution` and `Engine.RunPrepared` GoDoc](../../engine.go)
for the exact lifecycle, ownership, cancellation, capacity, timing, and
failure contracts.
## Inspect A Prompt
`Engine.InspectPrompt` resolves one prompt ID and optional version through the
engine's configured prompt source:
```go
inspection, err := engine.InspectPrompt(ctx, "report.summary", "")
```
The result includes prompt identity, the opaque prompt hash, declared default
profile ID, declared input metadata, and normalized output contract. It
structurally loads the selected definition and referenced message content but
does not resolve a profile, load schemas or artifacts, render templates,
reserve capacity, or contact a model.
Use inspection for exact configuration checks and metadata discovery. Use
`PrepareExecution` rather than relying on a prior inspection when later
execution must freeze one exact source state, because filesystem-backed
inspection is only a point-in-time lookup.
See the
[prompt-inspection consumer guide](../consumers/pkg-promptkit.md#inspect-a-prompt-before-preparation)
and [`Engine.InspectPrompt` GoDoc](../../engine.go) for exact selection,
ownership, and error behavior.
## Inspect A Profile
`Engine.InspectProfile` resolves one explicit profile independently of a
prompt:
```go
inspection, err := engine.InspectProfile(ctx, "report-production")
```
The result includes the resolved effective execution target and whether a
later request must provide a direct credential. Environment-variable names may
be reported, but inspection does not read credential values or require the
named variable to be populated.
Inspection applies the engine's profile source precedence and resolves any
selected backend. It does not load a prompt, render content, reserve capacity,
or contact a model.
See the
[profile-inspection consumer guide](../consumers/pkg-promptkit.md#inspect-a-profile-before-prompt-work)
and [`Engine.InspectProfile` GoDoc](../../engine.go) for the exact resolution,
credential, ownership, and error contracts.
## Identify Capacity-Rejected Backends
Calls rejected at Promptkit's bounded engine admission boundary continue to
match `ErrCapacityExceeded`. Consumers can additionally obtain the selected
registered backend ID without parsing diagnostic text:
```go
result, err := engine.Run(ctx, request)
if errors.Is(err, promptkit.ErrCapacityExceeded) {
var capacityErr *promptkit.CapacityError
if errors.As(err, &capacityErr) {
// Record capacityErr.BackendID using application-owned diagnostics.
}
// Apply application-owned overload or retry policy.
}
```
The structured error applies to `Run` and `RunPrepared` admission rejection.
It does not represent provider throttling, quota exhaustion, cancellation
while waiting for generation capacity, or another model-client failure.
Promptkit does not prescribe retry timing or transport status mapping.
See the
[error-handling consumer guide](../consumers/pkg-promptkit.md#handle-errors),
the [`CapacityError` GoDoc](../../capacity_error.go), and the
[`ErrCapacityExceeded` GoDoc](../../engine.go) for the canonical contracts.
## Public API Additions
The release adds:
- `Engine.PrepareExecution`;
- `Engine.RunPrepared`;
- `PreparedExecution`, including `Details`, `Discard`, `String`, and
`GoString`;
- `Engine.InspectPrompt`;
- `PromptInspection`;
- `PromptInputDefinition`;
- `Engine.InspectProfile`;
- `ProfileInspection`; and
- `CapacityError`.
No public API was removed.
## Consumer Action
None. Existing `v0.3.0` workflows may upgrade without adopting the new APIs.
Consumers that adopt prepared execution should discard unused handles.
Consumers that need backend-specific capacity diagnostics may add an
`errors.As` check while retaining their existing `errors.Is` classification.

125
docs/releases/v0.5.0.md Normal file
View File

@@ -0,0 +1,125 @@
# Promptkit v0.5.0
This supplemental changelog and migration guide summarizes the consumer-facing
changes from `v0.4.0` to `v0.5.0`. The annotated `v0.5.0` tag is the
authoritative release record. Exact current contracts belong to the linked
GoDoc and durable documentation.
## Summary
`v0.5.0` makes provider requests less prescriptive and adds an application
fallback layer for profile definitions:
- unset optional provider controls are omitted from OpenAI-compatible request
bodies instead of being populated with framework values; and
- `WithFallbackProfileFS` lets an application package profile defaults that
operators can override through the existing ordinary profile sources.
These changes let compatible providers apply their own model defaults while
giving applications stable embedded profile IDs without weakening operator
configuration precedence.
## Compatibility
The release adds one public function and removes no public declaration.
Existing source code should continue to compile.
There is one intentional behavior change: when no profile or runtime override
selects `top_p`, Promptkit no longer sends the former framework value of `1`.
It omits `top_p` and lets the provider choose its behavior. Unset
`temperature` and `max_tokens` are likewise omitted. Explicit nonzero profile
values and runtime values—including explicit runtime zero values—retain their
precedence and wire effect.
Consumers that relied on Promptkit always sending `top_p: 1` should add that
value to the relevant profile or runtime override before upgrading. Consumers
that did not rely on the implicit sampling value require no migration.
Application fallback profiles are opt-in. Engines that do not call
`WithFallbackProfileFS` retain the previous profile-source behavior.
## Upgrade
Update the module dependency with:
```sh
go get gitea.maximumdirect.net/eric/promptkit@v0.5.0
go mod tidy
```
Run the consuming project's ordinary and race-enabled tests after upgrading.
If request payloads or model behavior are asserted in fixtures, review them for
the optional-parameter omission described below.
## Omitted Optional Provider Controls
The built-in OpenAI-compatible client now includes `temperature`,
`max_tokens`, and `top_p` only when a profile or runtime override selects the
value. An explicit runtime zero remains present because runtime override
pointers distinguish zero from an unspecified value.
Promptkit's positive generation deadline remains a framework concern and is
not a provider request-body default. Required request fields, session IDs,
structured output, reasoning selection, credentials, and explicit extra
parameters retain their existing behavior.
See the [framework default and precedence reference](../formats.md#defaults-and-overrides),
the [`ExecutionTargetOverride` GoDoc](../../types.go), and the
[OpenAI-compatible request-body contract](../integrations/openai-compatible-chat.md#request-body)
for current details.
## Embedded Application Fallback Profiles
Applications can package ordinary profile YAML in an `fs.FS` and register it
as a fallback source:
```go
//go:embed profiles/*.yaml
var applicationProfiles embed.FS
engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: "prompts",
ProfileDir: operatorProfileDir,
},
promptkit.WithFallbackProfileFS(applicationProfiles, "profiles"),
)
```
Leave `operatorProfileDir` empty when no operator source is configured. A
configured ordinary source is authoritative: a matching definition overrides
the application fallback, while a read or validation failure remains an error
instead of silently reaching a lower layer.
Profile definitions resolve in this order:
1. in-memory profiles supplied with `WithProfiles`;
2. the ordinary configured source selected by `WithProfileFile`,
`WithProfileFS`, or `Config.ProfileDir`;
3. the application source supplied with `WithFallbackProfileFS`; and
4. Promptkit's embedded built-in profiles.
Only an absent profile ID falls through. Sources provide complete profiles and
do not merge fields. Loading remains lazy, and the new source uses the existing
strict profile YAML and credential rules.
See the
[embedded-default consumer guidance](../consumers/pkg-promptkit.md#supply-embedded-application-defaults),
the [`WithFallbackProfileFS` GoDoc](../../engine.go), and the
[profile source reference](../formats.md#source-and-profile-precedence) for
current details.
## Public API Changes
The release adds:
- `WithFallbackProfileFS`.
No public declaration was removed or changed.
## Consumer Action
- Review any workflow that depended on Promptkit's implicit `top_p: 1` and
configure the value explicitly when required.
- Optionally adopt `WithFallbackProfileFS` when an application should package
overridable profile defaults.
- Run consumer tests after updating the module dependency.

View File

@@ -1,237 +0,0 @@
# Backend-Specific Concurrency Management
**Status:** Complete.
## Purpose
This roadmap defines the scope and target end state for engine-local,
backend-specific concurrency management. It records the intended capability,
consumer value, and important policy choices.
This document is planning material, not a description of current behavior.
Current exported contracts remain owned by Go declarations and GoDoc, backend
registration guidance by the
[consumer guide](../consumers/pkg-promptkit.md#configure-a-local-openai-compatible-endpoint),
and
implemented orchestration by the
[internal runner document](../internal/runner.md).
## Motivation
Different model backends can sustain very different request loads. A local
network endpoint may need a small concurrency limit, while OpenRouter can
usually accept substantially more simultaneous work. Requiring every consumer
to build its own semaphores and queues would duplicate routing knowledge,
create inconsistent cancellation behavior, and make it easy for one caller to
bypass the intended backend limit.
Promptkit should own this coordination because it already resolves each run to
an engine-scoped backend identity and owns every model-generation call made by
the runner. Consumers should continue submitting ready-to-run requests through
the synchronous API, including concurrently from multiple goroutines, without
implementing their own backend scheduler.
The buffered queue is a safety boundary, not an ordinary throughput
restriction. Its primary purpose is to prevent a bug or unintended submission
loop from creating an unbounded in-memory backlog.
## Scope
The feature will add optional concurrency policy to registered backends and
coordinate `Run` calls against independent per-backend capacity pools.
Each policy has two distinct controls:
- an active-generation limit, which protects the backend from too many
simultaneous model requests; and
- a bounded waiting capacity, which protects the process from admitting an
unbounded backlog.
Concurrency policy belongs to a backend registration. It is not a profile
model parameter and cannot be overridden per run. Profiles select the policy
through their backend ID, while a profile or request endpoint override remains
in the selected backend's pool.
`Prepare` does not call a model and will remain outside concurrency admission.
## Defaults And Configuration
The built-in OpenRouter backend will use:
- an active-generation limit of 16; and
- a waiting capacity of 1024.
The waiting default is intentionally generous. Reaching it should indicate
abnormal submission pressure rather than normal application behavior.
Consumer-registered backends will remain unlimited unless the consumer
configures an active-generation limit. When a consumer enables a limit and
does not specify waiting capacity, the waiting capacity will default to 1024.
Consumers may configure a different bounded capacity, including zero when
they want no admitted backlog beyond the active-limit-sized run set.
The public representation must distinguish an omitted waiting capacity from
an explicit zero.
Endpoint-only profiles have no backend registration from which to obtain
policy and will remain unlimited. A future engine-wide or endpoint-keyed
policy can be considered separately if consumers demonstrate that need.
Invalid limits or capacities will fail engine construction as invalid
configuration. Policy values will be copied into engine-owned immutable state
along with the rest of the backend registration.
## Admission And Execution Behavior
`Run` remains a synchronous, wait-for-result operation. Concurrent callers may
block inside `Run` while waiting for their selected backend, then receive the
ordinary result or error from that invocation.
For a configured pool, the active-generation limit plus the waiting capacity
defines the maximum number of concurrent `Run` invocations that Promptkit will
accept for that backend. A waiting capacity of zero therefore accepts no more
runs than the active limit. Admission is immediate: a call either reserves one
of those bounded slots or receives the capacity error. An accepted run may
then wait internally for active-generation capacity.
For a limited backend, Promptkit will bound the number of accepted runs before
expensive artifact loading, prompt rendering, and large defensive copies where
practical. Lightweight prompt, profile, and backend resolution may occur first
when it is required to identify the correct capacity pool. This pre-admission
resolution must not become a second execution-precedence path with behavior
that can drift from `Prepare`.
An accepted run retains its admission until it completes or fails. Every
actual model-generation call for that run must separately observe the
backend's active-generation limit. This includes:
- the initial generation;
- every output-repair generation; and
- calls made through either the built-in or an injected model client.
Preparation and output validation should not hold an active-generation permit.
A repair remains part of its already-admitted run, but reacquires active
generation capacity so repairs cannot exceed the backend limit. It must not be
rejected merely because new runs filled the waiting queue after its initial
generation.
Within one backend pool, waiting generation calls should be served in FIFO
order, subject to canceled calls being removed. Different backend pools make
progress independently; a saturated local backend must not consume
OpenRouter's active or waiting capacity.
The feature will not promise ordering across backend pools or completion order
among admitted runs.
## Capacity Failure And Cancellation
When a backend's bounded waiting capacity is full, a new `Run` call will fail
promptly rather than waiting outside the bounded admission system. The public
API will expose a recognizable capacity-exhaustion error identity distinct
from invalid configuration, invalid requests, and model-client failures.
Rejected calls return no partial result and do not invoke the model client.
Waiting within the admitted backlog or for active-generation capacity must
honor the caller's context. Cancellation or deadline expiry will:
- stop waiting promptly;
- release any admission or generation capacity held by that invocation;
- preserve the applicable context error identity; and
- avoid invoking the model client if cancellation wins before generation
starts.
Capacity must also be released after preparation, generation, validation,
repair, or collaborator failure. One failed or canceled run must not reduce
the backend's future usable capacity.
Elapsed `Run` timing will include time spent waiting after the call is
accepted. `PreparedRun` timing will continue to describe preparation rather
than queue waiting.
## Engine And Client Boundaries
All pools and queued state belong to one `Engine`. Separate engines do not
share capacity, even when they register the same backend ID or endpoint. The
feature introduces no process-global scheduler.
The engine will apply policy consistently to the built-in model client and an
injected `LLMClient`. Consumers calling their own client outside Promptkit are
outside this boundary. Injected clients remain responsible for their internal
thread safety and cancellation behavior.
Backend policy is keyed by the resolved backend ID rather than endpoint text.
This preserves stable routing when a selected backend's endpoint is overridden
and avoids accidentally combining unrelated registrations that happen to use
the same URL.
## Queue Lifetime And Observability
Admission state is buffered, ephemeral, and in-process. It is not persisted
and has no survival guarantee across engine disposal or process termination.
Promptkit will not introduce background job ownership or require consumers to
start or stop workers.
The initial feature does not require public queue-depth metrics, callbacks, or
inspection APIs. Capacity errors and ordinary call timing provide the
consumer-visible behavior. Operational observability can be added later
without coupling the scheduling mechanism to an application logging or
metrics system.
## Compatibility
Consumer-registered backends and endpoint-only profiles remain unlimited
unless concurrency is explicitly configured, preserving their existing
behavior.
The built-in OpenRouter backend will change from unlimited concurrency to a
limit of 16 with a bounded waiting capacity of 1024. Ordinary synchronous
calls remain unchanged, while unusually high concurrent use may now wait or
return the capacity error. This behavioral change must be identified in the
release notes for the version that publishes it.
Adding backend policy fields and a public capacity error is otherwise
additive. The change will use a pre-`v1` minor release under Promptkit's
[release policy](../release.md#release-model).
## Non-Goals
This scope does not include:
- asynchronous job handles, polling, or detached result delivery;
- durable or cross-process queues;
- persistence or recovery across engine or process shutdown;
- priorities, scheduling weights, or consumer-defined fairness classes;
- automatic retries, backoff, rate-limit interpretation, or provider quota
discovery;
- token-per-minute or request-per-minute rate limiting;
- dynamic reconfiguration after engine construction;
- per-profile or per-run concurrency overrides;
- endpoint-keyed pooling for profiles without a backend ID;
- process-global coordination across engines;
- application worker lifecycle, logging, tracing, or metrics policy; or
- changes to prompt, profile, schema, or model-provider wire formats.
## Target End State
This roadmap reaches its target end state when:
- each engine independently coordinates configured backend capacity;
- the built-in OpenRouter backend allows 16 active generations and up to 1024
waiting runs;
- consumer backends can opt into their own active and waiting limits while
remaining unlimited by default;
- endpoint overrides retain the selected backend's capacity pool and
endpoint-only profiles remain unlimited;
- synchronous `Run` callers wait for and receive their ordinary result;
- admission is bounded before expensive preparation work where practical;
- every initial and repair generation observes the backend's active limit
without serializing preparation or validation;
- a full waiting queue returns a recognizable capacity error without invoking
the model client;
- cancellation and all failure paths promptly release capacity and preserve
context error identity;
- built-in and injected model clients receive the same scheduling behavior;
- pools remain ephemeral, engine-scoped, and independent across backend IDs;
and
- current-state GoDoc, consumer, internal, and release documentation describe
the implemented behavior once it lands.

82
docs/roadmap/deferred.md Normal file
View File

@@ -0,0 +1,82 @@
# Deferred Feature Ideas
## Purpose
This document catalogs feature ideas that remain potentially useful but have
been deliberately postponed. These ideas are not awaiting ordinary selection
from the [future feature catalog](future.md); each has a stated reason to wait
and should be reconsidered only when its trigger becomes relevant.
Deferred entries are not commitments, schedules, active implementation plans,
or descriptions of current behavior. When an entry is reactivated, move it to
`future.md` for evaluation or directly into a focused roadmap after its open
design dependencies have been resolved.
## Deferred Ideas
### Semantic Execution-Target Fingerprints
**Reason for deferral:** A stable digest requires a deliberate semantic-
equality and versioning design. Notarius can safely use conservative source
hashes and a Promptkit release marker today, while Weatherreporter does not
currently reuse LLM-dependent checkpoints.
Promptkit could expose an opaque equality value for a resolved profile and its
effective generation target. This would let checkpointing consumers detect
generation-affecting configuration changes without hashing YAML presentation
or depending on Promptkit's built-in catalog layout.
The digest should change with semantically relevant state such as the resolved
model, endpoint, backend routing identity, request defaults, extra parameters,
profile generation settings, and selected built-in profile semantics. It
should exclude credential values, concurrency and queue policy, source paths,
comments, formatting, and other representation-only changes. Whether a
credential environment-variable name affects equality must be decided
explicitly. The encoding should remain opaque and internally versioned so
Promptkit can deliberately invalidate earlier digests when its resolution
semantics change.
Reconsider this idea when a downstream consumer needs Promptkit-owned
checkpoint equality or when a broader semantic identity design is selected.
### Eager Source Validation
**Reason for deferral:** Exact prompt and profile inspection may already
provide a sufficiently small validation surface. Experience from downstream
adoption should establish whether an engine-wide operation would add enough
value to justify its broader contract.
Promptkit could provide an explicit offline operation that discovers and
structurally validates configured prompt, profile, and schema sources without
model generation. The normal `NewEngine` path would remain lazy.
An eager operation would need coherent handling for duplicate prompt IDs and
versions, strict YAML decoding, referenced content files, profile/backend
membership, schema syntax and transitive references, context cancellation,
and source-specific public errors. Credential declarations must remain
separate from credential values; checking current environment availability,
if supported at all, should be an explicit option and must not expose secrets.
Reconsider this idea after downstream use of `InspectPrompt`,
`InspectProfile`, and fixture-based preparation demonstrates a concrete gap.
### Structured Generation Errors
**Reason for deferral:** Existing `ErrLLMGenerate` classification, preserved
injected-client errors, and prepared execution details currently provide the
necessary failure boundary. A typed error should wait for stronger downstream
demand and a transport-neutral field design.
Promptkit could expose safe structured generation context through
`errors.As` while preserving `errors.Is(err, ErrLLMGenerate)`. Potential
fields include the selected backend ID and model plus an optional HTTP status
when the built-in OpenAI-compatible transport supplies one.
The design must not expose provider response bodies, endpoints, credential
environment names or values, request content, or generated content. It should
not duplicate prompt and profile provenance already available from a prepared
execution, and it must preserve the identity of errors returned by injected
clients. Retry and backoff policy remains a consumer responsibility.
Reconsider this idea when consumers need structured generation diagnostics
beyond the existing sentinel, wrapped client error, and preparation record.

View File

@@ -12,6 +12,9 @@ consumer value, and important scope boundaries. Defer API design,
implementation details, sequencing, and acceptance criteria until an idea is
selected.
Ideas that have been deliberately postponed rather than left available for
ordinary selection belong in the [deferred catalog](deferred.md).
## Using This Catalog
- Add an idea when its purpose and likely value can be stated clearly.
@@ -23,6 +26,8 @@ selected.
- When an idea is selected, move its active planning to a focused roadmap or,
when it requires a durable architectural decision, an ADR. Update
current-state documentation only when implementation lands.
- Move an idea to `deferred.md` when maintainers decide to retain it but wait
for a stated design dependency, demand signal, or reconsideration trigger.
- Remove ideas that are no longer relevant. Retain a rejected idea only when
its rationale is likely to prevent repeated reconsideration.
@@ -33,36 +38,7 @@ consumers.
## Ideas
Prompt-independent profile inspection has been selected for active planning in
the [focused feature roadmap](profile-inspection.md). The remaining ideas are
still available for future selection.
### Prompt-definition inspection
Provide exact prompt-definition lookup without rendering, placeholder inputs,
profile resolution, or model generation, as requested by
[Weatherreporter](weatherreporter-promptkit-wishlist.md#priority-2-prompt-definition-inspection).
- Return caller-owned identity, version, input definitions, default-profile,
output-contract, and opaque definition-equality information.
- Apply ordinary prompt-source precedence and exact ID/version selection.
- Validate the selected definition and referenced prompt content
structurally, without returning source bodies or rendered messages.
- Leave complete cross-source corpus validation and enumeration outside the
initial inspection contract.
### Structured capacity errors
Add safe structured context to backend admission rejection, as requested by
[Notarius](notarius-promptkit-wishlist.md#priority-4-structured-capacity-errors)
and
[Weatherreporter](weatherreporter-promptkit-wishlist.md#structured-capacity-errors).
- Preserve compatibility with `errors.Is(err, ErrCapacityExceeded)`.
- Support `errors.As` to obtain the stable backend ID.
- Do not expose endpoints, credential configuration or values, request
content, or speculative retry timing.
- Keep retry and backoff policy with downstream consumers.
No ideas currently await selection.
## Entry Format

View File

@@ -1,670 +0,0 @@
# Prompt-Independent Profile Inspection Implementation Plan
**Status:** Ready for implementation.
## Purpose
This document is the decision-complete implementation plan for
[prompt-independent profile inspection](profile-inspection.md). It is written
for a coding agent that will implement each stage in order.
The feature roadmap owns the motivation, consumer workflow, policy choices,
compatibility requirements, non-goals, and target end state. This document
owns the concrete design, file-level changes, implementation sequence, test
ownership, documentation work, validation commands, and completion gates.
## Implementation Rules
- Complete the stages in order. Keep the repository compiling and the focused
tests passing at every stage boundary.
- Preserve unrelated working-tree changes. In particular, retain the accepted
feature roadmap and the corresponding future-roadmap and downstream-wishlist
edits that may already be uncommitted.
- Follow every policy under `docs/policy/`, the task-specific reading guide in
`docs/development.md`, and the accepted behavior in
`profile-inspection.md`.
- Keep the supported API in the root `promptkit` package. Profile repositories,
backend resolution, and target assembly remain below Go's `internal/`
boundary.
- Reuse the exact profile-source, backend-membership, and execution-target
precedence used by `Prepare`, `PrepareExecution`, and `Run`. Do not create a
second profile loader, backend registry, or target-merging implementation.
- Preserve the observable behavior and error ordering of existing preparation
and execution methods. The new inspection operation must not require a
prompt, prompt default profile, request override, credential value, capacity
admission, model client, renderer, artifact reader, schema source, or
validator.
- Treat structural validation as the existing profile, backend, and effective
target invariants. Do not add new endpoint URL policy, provider
connectivity checks, reserved-extra-parameter policy, credential syntax, or
file-format validation rules as part of this feature.
- Never read, retain, return, format, or log an environment credential value.
The result may contain only the effective environment-variable name and the
direct-key-required boolean.
- Return caller-owned public values. In particular, nested
`ExecutionTarget.ExtraParams` maps, slices, and objects must not alias
engine-owned state or another inspection result.
- Keep tests lean and behavior-focused. Reuse existing profile repository,
backend registry, target precedence, and preparation tests rather than
duplicating their complete matrices.
- Update exact contracts in GoDoc with the exported declarations. Update
current-state consumer, format, and internal documentation only after the
corresponding code exists.
- Do not add release notes, change a module version, create a release, or tag a
commit as part of this work.
## Fixed Design
### Public API
Add this root-package value immediately after `ExecutionTarget` in `types.go`:
```go
// ProfileInspection is the caller-owned result of Engine.InspectProfile.
// The exact GoDoc is specified below.
type ProfileInspection struct {
ProfileID string
EffectiveModelParams ExecutionTarget
APIKeyRequired bool
}
```
Add this method to `Engine` in `engine.go`, immediately before `Prepare`:
```go
func (e *Engine) InspectProfile(
ctx context.Context,
profileID string,
) (*ProfileInspection, error)
```
Do not accept `RunRequest`, `ExecutionTargetOverride`, an API key, an
environment override, or options on this method. Exact inspection of one
explicit profile ID is the complete operation.
`ProfileInspection` has no stable JSON contract. Do not add JSON tags,
`MarshalJSON`, `UnmarshalJSON`, or a custom string representation. Ordinary Go
encoding of the exported fields is not prohibited, but consumers must not be
promised compatibility for that encoding.
The exact type and method GoDoc must establish:
- surrounding whitespace is trimmed from `profileID`; the resulting nonblank
ID is looked up exactly and case-sensitively;
- the result applies the engine's ordinary in-memory, configured-source, and
built-in profile precedence;
- `EffectiveModelParams` contains framework defaults overlaid by the selected
backend and then the selected profile, with no request override;
- `EffectiveModelParams.BackendID` is empty for endpoint-only profiles;
- `EffectiveModelParams.APIKeyEnv` is an environment-variable name and never
its value;
- `APIKeyRequired` means a direct request credential is required and is
mutually exclusive with a nonblank effective `APIKeyEnv`;
- the method never derives an ID from a prompt's `default_profile`;
- the method performs no prompt lookup, rendering, artifact or schema work,
backend admission, provider connectivity check, or model generation;
- credential availability is not checked, so an absent or blank named
environment variable is not an error;
- the returned value and all nested mutable values are caller-owned;
- filesystem-backed inspection is a point-in-time lookup and does not freeze
a profile for a later execution;
- a nil engine matches `ErrInvalidConfig`;
- a blank ID matches `ErrInvalidRequest`;
- an absent exact ID matches `ErrProfileNotFound` and not `ErrProfileLoad`;
- malformed or unreadable profile data, an unknown backend, or an invalid
resolved target matches `ErrProfileLoad`;
- cancellation observed during profile loading matches `ErrProfileLoad` while
preserving the context error through `errors.Is`; and
- the method returns no partial result on error.
Update the `Engine` type GoDoc to include `InspectProfile` among the operations
safe for concurrent calls. Do not imply that injected collaborators become
safe when their existing contracts do not provide that guarantee.
Update `doc.go` in the same stage:
- include `Engine.InspectProfile` in the package's operation list;
- include inspection in the concurrency and caller-ownership summary; and
- list `ProfileInspection` among construction and inspection values without a
stable JSON representation.
Do not add `ProfileInspection` to the stable JSON list.
### Internal Result
Add this internal value to `internal/domain/domain.go` near
`ExecutionProfile` and `ExecutionTarget`:
```go
// ProfileInspection is the resolved result of exact profile inspection.
type ProfileInspection struct {
ProfileID string
EffectiveModelParams ExecutionTarget
APIKeyRequired bool
}
```
Do not add JSON or YAML tags. The internal value is a use-case result, not a
file format or persistence contract.
Add a root conversion in `convert.go`:
```go
func fromDomainProfileInspection(
inspection *domain.ProfileInspection,
) *ProfileInspection
```
The conversion must:
- return `nil` for a nil internal value;
- copy the scalar fields;
- convert the target through the existing `fromDomainExecutionTarget`; and
- therefore use the existing recursive `copyAnyMap` boundary for nested extra
parameters.
Do not add `APIKeyRequired` to the public `ExecutionTarget`. Keeping the
requirement on `ProfileInspection` preserves the existing stable JSON and
general execution-target contract.
### Shared Profile Selection
Add `internal/usecase/profile_inspection.go`. Define one private selection
value:
```go
type resolvedProfileSelection struct {
id string
profile *domain.ExecutionProfile
backend *domain.Backend
}
```
Add a private runner helper:
```go
func (r *Runner) resolveProfileSelection(
ctx context.Context,
profileID string,
) (*resolvedProfileSelection, error)
```
The helper must perform these operations in order:
1. trim surrounding whitespace from `profileID`;
2. reject a blank result with `ErrInvalidRequest`;
3. reject a nil runner profile repository with `ErrProfileLoad` rather than
panicking;
4. call the existing `profile.Repository.GetProfile` exactly once;
5. wrap every repository error with `ErrProfileLoad` while preserving the
underlying error with `%w`;
6. reject a nil profile returned without an error as `ErrProfileLoad`;
7. make a value copy of the selected profile before normalization so the
runner does not mutate repository-owned state;
8. trim the copied profile's backend ID;
9. when that ID is nonblank, resolve it exactly once through the existing
`BackendResolver`, preserving the current unknown-backend
`ErrProfileLoad` wrapping and backend ID context; and
10. return the normalized exact ID, copied profile, and optional defensive
backend value.
Do not inspect all sources, enumerate profiles, expose a source path, fall back
after a malformed higher-precedence match, or infer a backend from an endpoint
or model.
Refactor the profile-loading and backend-resolution block in
`Runner.resolvePreparation` to call `resolveProfileSelection`. Leave prompt
selection, prompt default-profile selection, prompt hashing, target overrides,
request credentials, output-contract resolution, artifact work, and all later
ordering where they currently occur.
This refactor must preserve:
- explicit request profile selection over prompt `default_profile`;
- the existing `ErrProfileRequired` result when neither exists;
- exact repository and backend lookup behavior;
- existing public not-found versus profile-load classification;
- request override precedence and target-presence tracking;
- credential checking during ordinary preparation; and
- current `Prepare`, `PrepareExecution`, `Run`, and `RunPrepared` behavior.
### Shared Target Resolution And Structural Validation
Continue using the existing `resolveExecutionTarget` for framework, backend,
profile, and optional request precedence. Do not duplicate or move the
individual merge rules into the inspection code.
Extract the two existing effective-target requiredness checks from
`resolvePreparation` into a private pure helper:
```go
func validateResolvedExecutionTarget(
target domain.ExecutionTarget,
) error
```
It checks only that the trimmed endpoint and model are nonblank and returns a
plain descriptive error. It does not validate URL syntax, contact the
endpoint, validate credentials, or introduce new parameter rules.
`resolvePreparation` calls this helper at the same current point: after
request override resolution and direct API-key assignment, and before
`validateAPIKey`. It wraps a failure with `ErrInvalidRequest`, preserving
ordinary preparation behavior.
Add the internal inspection method:
```go
func (r *Runner) InspectProfile(
ctx context.Context,
profileID string,
) (*domain.ProfileInspection, error)
```
It performs these operations:
1. trim and reject a blank ID as `ErrInvalidRequest`;
2. if the supplied context is already canceled, return an error wrapping both
`ErrProfileLoad` and `ctx.Err()` without touching the repository;
3. call `resolveProfileSelection`;
4. call `resolveExecutionTarget` with the selected backend, selected profile,
and a nil request override;
5. wrap any target-resolution error with `ErrProfileLoad`;
6. call `validateResolvedExecutionTarget` and wrap a failure with
`ErrProfileLoad`;
7. defensively clear `target.APIKey`;
8. copy `target.APIKeyRequired` into the result's `APIKeyRequired`; and
9. return the normalized profile ID and effective target.
Do not call `validateAPIKey`, `os.Getenv`, prompt repositories, artifact
readers, renderers, validators, capacity admission, output repair, or the
model client. Do not populate request target-presence state.
The internal `ExecutionTarget` may retain its private `APIKeyRequired` field;
the root conversion intentionally omits that private field from the public
target and publishes the separate inspection boolean.
### Root Facade And Error Mapping
`Engine.InspectProfile` follows the existing facade pattern:
1. reject a nil engine or nil runner with `ErrInvalidConfig`;
2. pass the context and string ID directly to `Runner.InspectProfile`;
3. map internal failures through the existing `mapPublicError`; and
4. convert a successful result with `fromDomainProfileInspection`.
No request conversion is needed. Do not add a public sentinel or typed error.
The existing error mapping already has the required ordering:
- an underlying `profile.ErrProfileNotFound` maps to
`ErrProfileNotFound` before the enclosing use-case `ErrProfileLoad` is
considered;
- other use-case `ErrProfileLoad` failures map to `ErrProfileLoad`; and
- `usecase.ErrInvalidRequest` maps to `ErrInvalidRequest`.
Do not reorder or otherwise change `errors.go` unless a focused test proves
that the existing mapping does not meet this plan. Preserve underlying
repository, backend, and context errors through `errors.Is`.
### Credentials, Ownership, And Concurrency
Inspection has no input through which a direct credential value can enter.
Backend and file-profile `APIKeyEnv` values remain names only. An in-memory
profile with `APIKeyRequired` clears an inherited backend environment name
through the existing target merge behavior.
The implementation must succeed for:
- a backend or profile naming an unset environment variable;
- a backend or profile naming an environment variable whose value is blank;
- an in-memory profile requiring a direct key when no key is supplied; and
- a profile requiring no credential.
The result expresses these effective states:
| `EffectiveModelParams.APIKeyEnv` | `APIKeyRequired` | Meaning |
| --- | --- | --- |
| nonblank | `false` | The profile resolves to the named environment source. |
| blank | `true` | A later execution must supply a direct key or explicit request environment override. |
| blank | `false` | The resolved target declares no credential requirement. |
The implementation must never produce a successful public result with both a
nonblank `APIKeyEnv` and `APIKeyRequired == true`.
Public ownership is enforced at the root conversion boundary. A consumer may
mutate the returned target and arbitrarily nested JSON-compatible extra
parameters without affecting:
- the engine registry or profile repository;
- a later `InspectProfile` call;
- `Prepare`, prepared execution, or `Run`; or
- another result already returned to a caller.
No new mutable engine state, cache, global registry, lock, or goroutine is
needed. Concurrency safety follows from the existing immutable registry and
repository contracts plus per-call values.
### Test Ownership
Add focused internal tests in
`internal/usecase/profile_inspection_test.go`. Use small repository and backend
fakes already present in the package where practical; do not build a parallel
fixture framework.
The internal tests own:
- blank-ID rejection;
- pre-canceled context classification without repository access;
- one exact profile and backend lookup;
- framework-default, backend, and profile target precedence through the
existing resolver;
- endpoint-only behavior;
- effective environment-name reporting without availability checks;
- direct-key-required behavior clearing an inherited backend environment
name;
- missing-profile and unknown-backend wrapping; and
- nil repository, nil returned profile, and invalid resolved target defenses
only if these cases are not already cheaply covered through existing runner
fakes.
Keep the internal matrix compact. Profile parser tests continue to own YAML,
duplicates, raw-key rejection, ranges, and source discovery. Backend tests
continue to own registration validation. Existing target tests continue to
own every merge field and numeric override boundary.
Add external-package public contract tests in `public_contract_test.go`. They
own:
- the exported method and result shape through normal Go use;
- a nil engine returning `ErrInvalidConfig`;
- blank, missing, malformed, and unknown-backend public error identities,
including that not-found does not also match `ErrProfileLoad`;
- a pre-canceled inspection preserving both `ErrProfileLoad` and the context
error without consulting the profile repository;
- successful inspection with an absent credential environment value;
- no prompt dependency, demonstrated with an empty configured prompt
`fs.FS`;
- no model invocation, using the existing deterministic fake client;
- the three credential-requirement result states;
- an endpoint-only profile's empty backend ID;
- deep caller ownership of nested extra parameters across repeated
inspections; and
- representative equivalence between `InspectProfile.EffectiveModelParams`
and `Prepare.EffectiveModelParams` for the same engine, profile, and source
state with no request execution override.
Combine closely related assertions into a few readable behavioral tests.
Do not add a JSON golden test because `ProfileInspection` deliberately has no
stable JSON contract. Do not duplicate the complete repository-precedence,
target-field, parser-error, or credential-execution suites at the root layer.
Existing tests that must continue passing without semantic edits include:
- profile repository and built-in fallback tests;
- backend registry defensive-copy and lookup tests;
- execution-target precedence tests;
- `Prepare` and `Run` profile/backend equivalence tests;
- prepared-execution snapshot and credential tests; and
- public error mapping tests.
### Documentation Ownership
After the implementation and public tests pass, update current-state
documentation:
- `docs/consumers/pkg-promptkit.md`: add a concise task-oriented section near
profile selection showing `Engine.InspectProfile`, explaining when to use it
instead of a synthetic `Prepare`, how to interpret `APIKeyEnv` and
`APIKeyRequired`, and that credential enforcement timing remains
application policy. Link to the exact GoDoc rather than restating every
error and field contract.
- `docs/formats.md`: update profile selection and backend-membership wording
so exact inspection is recognized as another consumer of the existing
source and profile precedence. Do not redefine the exported method here.
- `docs/internal/runner.md`: describe the shared profile-selection boundary
and explain that inspection stops after structural target resolution,
before credential availability and all prompt-dependent work.
- `docs/internal/sources.md`: record that exact profile inspection performs
one point-in-time profile-source lookup without reading prompt, input, or
schema sources, and replace any ambiguous use of “inspection value” for
`PreparedRun` with “preparation value.”
- `docs/internal/overview.md`: add profile inspection to the existing root
facade and `internal/usecase` responsibility descriptions. Do not add a new
component or package row.
- `docs/roadmap/future.md`: remove the statement that profile inspection is in
active planning and leave the remaining unselected ideas intact.
- `docs/roadmap/notarius-promptkit-wishlist.md` and
`docs/roadmap/weatherreporter-promptkit-wishlist.md`: change the profile
inspection disposition from accepted planning to implemented behavior and
link to the durable consumer guidance or GoDoc rather than treating the
roadmap as current-state documentation.
- `docs/roadmap/profile-inspection.md`: change its status to `Complete` only
after the code, tests, current-state documentation, and full validation are
complete.
- `docs/roadmap/implementation.md`: change its status to `Complete` only after
every completion gate in this plan is satisfied.
Do not update release guidance in this feature implementation. A later release
pass decides whether the change warrants a supplemental release document.
## Stage 1: Implement Shared Internal Profile Resolution
### Objective
Add the internal inspection result and operation, share profile/backend
selection and structural target validation with ordinary preparation, and
prove the internal behavior without publishing the root API yet.
### Implementation Prompt
Implement only Stage 1 of
`docs/roadmap/implementation.md`. Read the complete feature roadmap and the
implementation rules and fixed design above before editing.
1. Add `domain.ProfileInspection` to `internal/domain/domain.go` without
serialization tags.
2. Add `internal/usecase/profile_inspection.go` with
`resolvedProfileSelection`, `resolveProfileSelection`,
`validateResolvedExecutionTarget`, and `Runner.InspectProfile` exactly as
specified.
3. Refactor `Runner.resolvePreparation` in `internal/usecase/runner.go` to use
the shared selection and target-validation helpers while preserving its
current ordering, error classification, target overrides, credentials, and
results.
4. Add lean behavioral tests in
`internal/usecase/profile_inspection_test.go`.
5. Run the focused validation below and repair regressions before ending the
stage.
Do not add the root public type or method, update current-state documentation,
or alter profile formats, backend registration, credential availability,
capacity, model-client, or provider behavior in this stage.
### Focused Validation
Run:
```sh
gofmt -w internal/domain/domain.go \
internal/usecase/profile_inspection.go \
internal/usecase/profile_inspection_test.go \
internal/usecase/runner.go
go test ./internal/usecase ./internal/profile ./internal/profile/builtin \
./internal/backend
go test ./internal/usecase -run \
'TestRunner(InspectProfile|Prepare|Run|PrepareExecution|RunPrepared)'
go vet ./internal/usecase ./internal/profile ./internal/backend
```
### Completion Gate
Stage 1 is complete only when:
- inspection reaches profile and backend resolution without any prompt or
execution collaborator;
- profile/backend selection is shared with ordinary preparation;
- target merging and requiredness checks are shared rather than duplicated;
- inspection does not check credential values or capacity;
- internal errors preserve the required identities;
- the ordinary preparation and execution tests still pass without weakened
assertions; and
- no root public API or current-state documentation claims the feature yet.
## Stage 2: Publish The Root Facade And Public Contract
### Objective
Expose the minimal caller-owned inspection API through `Engine`, preserve
public error and JSON compatibility, and protect the consumer-visible
contract.
### Implementation Prompt
Implement only Stage 2 of
`docs/roadmap/implementation.md` after Stage 1 satisfies its completion gate.
Re-read the fixed public API, error, credential, ownership, and test sections
before editing.
1. Add `ProfileInspection` and its exact GoDoc to `types.go` immediately after
`ExecutionTarget`. Do not add JSON tags or change `ExecutionTarget`.
2. Add `fromDomainProfileInspection` to `convert.go` using
`fromDomainExecutionTarget`.
3. Add `Engine.InspectProfile` and its exact GoDoc to `engine.go` immediately
before `Prepare`.
4. Update `Engine` GoDoc to include concurrent inspection.
5. Update the package GoDoc in `doc.go` for operation discovery, concurrency,
ownership, and the non-stable JSON classification.
6. Add compact external-package contract coverage in
`public_contract_test.go`, using existing fixtures and fakes where they
remain clear.
7. Confirm that the existing `errors.go` mapping meets the plan; do not change
it unless a required public identity test fails for a genuine mapping
reason.
8. Run the focused validation below and repair regressions before ending the
stage.
Do not add enumeration, request overrides, profile fingerprints, JSON
stability, prompt inspection, environment credential checks, caching, or
current-state prose documentation in this stage.
### Focused Validation
Run:
```sh
gofmt -w doc.go types.go convert.go engine.go public_contract_test.go
go test .
go test ./internal/usecase
go test . -run 'Test(InspectProfile|.*Profile.*PublicError|.*Profile.*Contract)'
go vet .
go build .
```
If the repository's actual focused test names differ, use the implemented test
names rather than weakening or skipping the intended assertions.
### Completion Gate
Stage 2 is complete only when:
- a consumer can call `Engine.InspectProfile` through the root package;
- the result contains only the normalized ID, caller-owned effective target,
and direct-key-required signal;
- no credential value can enter the result;
- environment, direct, and no-credential states are unambiguous;
- nil, blank, missing, malformed, unknown-backend, and cancellation errors
have the required public identities;
- not-found remains distinct from profile-load failure;
- the effective target matches ordinary preparation absent request overrides;
- repeated calls and caller mutation cannot alter engine-owned state;
- no stable JSON or new error contract was introduced; and
- existing public preparation, execution, and stable JSON tests remain
unchanged and passing.
## Stage 3: Update Documentation And Validate The Repository
### Objective
Make implemented profile inspection discoverable in its canonical
documentation, reconcile temporary roadmap state, and complete full repository
validation.
### Implementation Prompt
Implement only Stage 3 of
`docs/roadmap/implementation.md` after Stages 1 and 2 satisfy their completion
gates.
1. Update `docs/consumers/pkg-promptkit.md`, `docs/formats.md`,
`docs/internal/runner.md`, `docs/internal/sources.md`, and
`docs/internal/overview.md` according to the documentation ownership
section above.
2. Update the future catalog and both downstream wishlist dispositions so they
no longer describe profile inspection as merely accepted work.
3. Check every changed Markdown link and confirm its file and heading target.
4. Run the full validation sequence below.
5. Only after every check passes, set the feature roadmap and this
implementation plan to `**Status:** Complete.`
6. Re-run `git diff --check` after the status edits.
Do not add release notes, an example program, a new public package, or a
duplicate API reference. Keep detailed contracts in GoDoc and task-oriented
usage in the consumer guide.
### Full Validation
Run from the repository root:
```sh
gofmt -w internal/domain/domain.go \
internal/usecase/profile_inspection.go \
internal/usecase/profile_inspection_test.go \
internal/usecase/runner.go \
doc.go types.go convert.go engine.go public_contract_test.go
gofmt -l $(git ls-files '*.go')
go test ./...
go test -race ./...
go vet ./...
go build ./...
go run ./examples/go-library/prepare
git diff --check
git status --short
```
The `gofmt -l` command must print no paths. The maintained example must remain
offline and must not require a real credential or provider.
Inspect the final diff and confirm:
- only files required by this feature and pre-existing user changes are
present;
- no credential values, private infrastructure details, workspace files,
local replacements, generated binaries, or unrelated formatting changes
were added;
- the public declarations and GoDoc own exact API behavior;
- current-state documentation describes only implemented behavior and links
to canonical owners;
- roadmap documents contain future scope or completion status rather than a
duplicate current API reference; and
- no release, commit, or tag was created.
### Completion Gate
The implementation is complete only when:
- every Stage 1 and Stage 2 gate remains satisfied;
- the complete ordinary and race-enabled suites pass;
- vet, build, formatting, the maintained offline example, Markdown links, and
whitespace checks pass;
- consumer, format, internal, future, and wishlist documentation are
consistent with the implemented boundary;
- both roadmap statuses are `Complete`;
- the working tree contains no unintended files or changes; and
- the repository is ready for maintainer review without a commit or release
having been created by this plan.
## Open Questions
None. The accepted feature roadmap and the fixed decisions above fully specify
the implementation boundary.

View File

@@ -1,163 +0,0 @@
# Local Backend Convenience
**Status:** Complete.
## Purpose
Make the common case of using a local OpenAI-compatible endpoint concise and
easy to discover without introducing implicit configuration or a separate
backend abstraction.
The existing `Backend` type and registry remain the canonical, fully
configurable interface. A small convenience constructor will cover the usual
local-network case, while improved consumer documentation will make it clear
when an endpoint-only profile, the convenience constructor, or a complete
`Backend` value is appropriate.
## Motivation
Consumers can already use a local endpoint by setting `Profile.Endpoint`, or
register one as a backend with `WithBackend`. The first option is concise but
does not provide shared backend-level concurrency control. The second supports
the complete backend feature set but requires consumers to understand and
populate several fields for a common configuration.
Most consumers adding a local backend need only:
- a stable backend ID;
- an OpenAI-compatible endpoint; and
- a concurrency limit appropriate for the local server.
Promptkit should provide a direct path for that case while keeping all
configuration explicit and preserving the full registry interface for
advanced needs.
## Consumer Paths
Documentation should present three progressively more configurable paths:
1. Set `Profile.Endpoint` when a profile only needs to target a local endpoint
and does not need shared backend policy.
2. Use the local-backend convenience constructor when profiles should share a
named local endpoint and its concurrency limit.
3. Construct a complete `Backend` value when the consumer needs a custom
backend ID, authentication, extra request parameters, an explicit queue
capacity, or multiple local backends.
These are complementary interfaces. The convenience constructor must return an
ordinary `Backend`, so it does not create a second configuration model.
## Public Convenience API
The public package should expose:
```go
const BackendLocal = "local"
func LocalBackend(endpoint string, concurrencyLimit int) Backend
```
`LocalBackend` should return a `Backend` with:
- `ID` set to `BackendLocal`;
- `Endpoint` set to the supplied endpoint;
- `ConcurrencyLimit` set to the supplied limit; and
- all other fields left at their zero values.
The returned value is passed to `WithBackend` and follows the same copying,
normalization, validation, and registration rules as any consumer-constructed
`Backend`.
The constructor should be a transparent value constructor. It should not read
environment variables, mutate global state, register the backend, validate
arguments independently, or create profiles. Consumers may inspect or modify
the returned value before registration, although documentation should direct
substantially customized configurations to the full `Backend` form.
## Identity and Registration
`BackendLocal` is a conventional ID used by the convenience constructor. It is
not pre-registered and should not become a specially reserved registry ID.
Consumers remain responsible for registering the returned backend with
`WithBackend` and naming it from profiles through `BackendID`.
This distinction preserves compatibility with consumers that may already
register their own backend using the ID `"local"`. Normal duplicate-ID rules
still apply if a consumer attempts to register more than one backend with that
ID.
Consumers that need multiple local endpoints should choose distinct IDs and
use complete `Backend` values rather than the single conventional helper ID.
## Concurrency and Queue Semantics
The constructor must preserve the existing backend concurrency contract:
- a positive concurrency limit bounds simultaneous requests and uses the
existing default queue capacity because `QueueCapacity` remains `nil`;
- a zero concurrency limit leaves the backend unconstrained; and
- a negative concurrency limit is rejected through the existing engine
configuration validation path.
The constructor should not select a hidden default concurrency limit. Local
servers vary substantially in capacity, so the consumer should make this
choice explicitly.
## Documentation
The final documentation state has two canonical surfaces:
- Public Go documentation describes the exact contract of
`BackendLocal` and `LocalBackend`, including their conventional,
non-pre-registered nature.
- The [promptkit consumer guide](../consumers/pkg-promptkit.md) includes
a task-oriented local-endpoint section that shows the three consumer paths,
explains the decision between them, and provides concise examples of the
endpoint-only and convenience-constructor forms.
The consumer guide continues to document the full `Backend` interface as
the advanced path rather than attempting to reproduce every configuration
variation through convenience APIs.
## Compatibility
This feature is additive:
- existing endpoint-only profiles continue to work unchanged;
- existing `Backend` values and `WithBackend` registrations remain the
canonical general-purpose interface;
- existing registrations using the literal ID `"local"` remain valid; and
- OpenRouter defaults and all other backend behavior remain unchanged.
No consumer is required to adopt the convenience constructor.
## Non-Goals
This work does not include:
- pre-registering or implicitly enabling a local backend;
- discovering a local endpoint, API key, or concurrency limit from environment
variables;
- adding local-backend fields to `Config`;
- selecting a default local model or creating a profile automatically;
- adding a combined backend-and-profile constructor;
- adding convenience parameters for API keys, extra request parameters, or
queue capacity;
- replacing or redesigning the backend registry;
- adding support for non-OpenAI-compatible local APIs; or
- changing backend routing, scheduling, or queue behavior.
## Target End State
After this work:
- consumers with a simple one-profile local endpoint can continue to configure
it directly on the profile;
- consumers needing a shared local endpoint and concurrency policy can express
it with one `LocalBackend` call and register the returned value normally;
- consumers with advanced or multiple-local-backend requirements have a clear
path to the complete `Backend` interface;
- all local configuration remains explicit, inspectable, and compatible with
dependency injection; and
- canonical documentation makes the simplest suitable interface easy to find
without obscuring the underlying registry model.

View File

@@ -1,358 +0,0 @@
# Notarius PromptKit Wishlist
## Purpose
This document records features and interface changes that would be useful
additions to PromptKit from the perspective of the maintainers of Notarius, a
downstream application that consumes PromptKit.
PromptKit v0.3.0 provides the capabilities Notarius currently needs. None of
the ideas below blocks current Notarius development. They are opportunities to
reduce downstream workarounds, improve integration correctness, and make
PromptKit more ergonomic for applications with configuration validation,
debugging, checkpointing, and operational-observability requirements.
The examples are API sketches intended to communicate the desired capability,
not prescriptive names or finalized Go contracts.
## Priority 1: Atomic Execution With Prepared Details
**Disposition:** Covered by the accepted
[executable preparation handles](prepared-execution.md) roadmap. The shared
two-phase capability should provide the required single-preparation
consistency; a separate `RunDetailed` method is not cataloged initially.
### Downstream need
Notarius needs both:
- the completed `RunResult`; and
- the rendered messages, effective output contract, hashes, and other
preparation details exposed by `PreparedRun`.
Notarius uses the prepared details to construct redaction-aware debug bundles
and retain enough information to diagnose model behavior.
### Current integration
Notarius currently calls `Engine.Prepare` and then `Engine.Run` with the same
request. Because `Run` performs preparation internally, a successful request
resolves and prepares the same work twice.
This duplicates profile resolution, input hashing, schema loading, and prompt
rendering. It also creates a theoretical consistency window in which a
filesystem-backed prompt, profile, schema, or input could change between the
explicit preparation and the preparation performed by `Run`.
### Requested capability
Add an opt-in execution method that prepares exactly once and returns both the
prepared details and completed result:
```go
type RunReport struct {
Prepared PreparedRun
Result RunResult
}
func (e *Engine) RunDetailed(
ctx context.Context,
req RunRequest,
) (*RunReport, error)
```
The exact names are flexible. The important contract is that preparation
occurs once and that the returned prepared state describes the execution that
produced the returned result.
Existing `Prepare` and `Run` behavior should remain available for consumers
that need only one side of the operation.
### Design considerations
- Keep this API additive and preserve the existing simple `Run` workflow.
- Return caller-owned copies under PromptKit's existing ownership rules.
- Define whether any prepared details are available after an operational
generation or validation error. Notarius does not require partial results
for the initial use case, but an explicit contract would be valuable.
- Preserve cancellation and backend-admission semantics.
- Do not add all prepared content directly to `RunResult`. Rendered prompt
content can be large and sensitive, and consumers should opt in to receiving
it.
### Value to Notarius
This is the highest-value wishlist item. It would remove duplicate work from
every successful PromptKit-backed call and ensure that retained debug material
corresponds atomically to the actual execution.
## Priority 2: Prompt-Independent Profile Inspection
**Disposition:** Covered by the accepted
[prompt-independent profile inspection](profile-inspection.md) roadmap.
### Downstream need
Notarius validates configured pipeline profile IDs before beginning a run. It
needs to determine whether:
- a profile exists;
- its referenced backend is registered;
- its execution target can be resolved; and
- it declares a credential requirement that the application may need to
enforce.
This validation should not require model generation.
### Current integration
Notarius constructs a synthetic prompt using `testing/fstest.MapFS`, supplies a
dummy transcript, and calls `Engine.Prepare` solely to exercise profile and
backend resolution. This works, but prompt preparation is serving as a
substitute for a profile-inspection interface.
### Requested capability
Add a prompt-independent profile-resolution API, for example:
```go
type ResolvedProfile struct {
ProfileID string
BackendID string
EffectiveTarget ExecutionTarget
APIKeyEnv string
}
func (e *Engine) ResolveProfile(
ctx context.Context,
profileID string,
) (ResolvedProfile, error)
```
The returned shape may differ, but it should provide enough information for a
consumer to validate an explicit profile selection without inventing a prompt
or supplying placeholder inputs.
### Design considerations
- Resolve built-in, file-backed, and programmatic profiles using normal
PromptKit precedence.
- Validate that a referenced backend registration exists.
- Do not resolve, retain, or expose credential values.
- Report credential requirements, such as an environment-variable name, so
the consuming application can decide whether availability is required at
configuration-validation time or only at execution time.
- Return caller-owned values.
- Preserve typed or sentinel error classification for missing and invalid
profiles.
- Consider accepting an `ExecutionTargetOverride` if consumers need to inspect
the same effective target that a run-level override would produce.
- Enumeration of all profiles is not required for the Notarius use case; exact
lookup by ID is sufficient.
### Value to Notarius
This would eliminate a synthetic production-only prompt fixture and establish
a direct, supported contract for configuration-time profile and backend
validation.
## Priority 3: Semantic Execution-Target Fingerprints
**Disposition:** Deferred until prompt-independent profile inspection defines
the resolved target whose configuration identity would be fingerprinted.
### Downstream need
Notarius checkpoints model-backed pipeline stages. A checkpoint must not be
reused when generation-affecting PromptKit configuration changes.
Notarius therefore needs a stable equality signal for the effective profile
and backend target used by a pipeline.
### Current integration
Notarius currently constructs this identity itself from:
- a manually maintained marker for the PromptKit release and built-in profile
catalog;
- raw hashes of configured profile files; and
- a separate hash of the configured conventional local-backend endpoint.
This is safe but conservative and coupled to PromptKit details. Raw file
hashing also invalidates checkpoints for semantically irrelevant YAML changes,
such as comments or formatting.
### Requested capability
Expose an opaque semantic digest for a resolved profile and its effective
generation target. It could be returned by the proposed profile-resolution
API:
```go
type ResolvedProfile struct {
ProfileID string
BackendID string
EffectiveTarget ExecutionTarget
ExecutionDigest string
}
```
Alternatively, PromptKit could expose a dedicated method such as
`ProfileExecutionDigest(profileID)`.
### Desired equality semantics
The digest should change when generation-affecting state changes, including:
- resolved model and endpoint;
- backend routing identity;
- backend request defaults and extra parameters;
- profile generation parameters; and
- the semantic identity of any selected built-in profile.
The digest should not incorporate:
- credential values;
- concurrency or queue capacity;
- filesystem source paths;
- YAML comments or formatting; or
- other settings that affect scheduling or source representation without
changing the generation target.
The credential environment-variable name may need to participate if changing
it can select a materially different provider account or target. PromptKit
should define this deliberately while continuing to exclude the resolved
secret value.
### Design considerations
- Treat the digest as an opaque equality value rather than a public encoding
of internal structures.
- Document which categories of change affect equality.
- Include a versioned semantic marker internally so PromptKit can deliberately
invalidate old digests when its resolution semantics change.
- Prefer a per-profile digest over a digest of every profile known to an
engine. Notarius generally knows which profiles a resolved pipeline uses.
- Do not require consumers to know PromptKit's built-in catalog version.
### Value to Notarius
This would let Notarius remove its PromptKit release marker and raw
profile-source fingerprinting, reduce unnecessary checkpoint invalidation, and
delegate execution-target equality to the component that owns target
resolution.
## Priority 4: Structured Capacity Errors
**Disposition:** Accepted into the
[future catalog](future.md#structured-capacity-errors).
### Downstream need
Notarius translates PromptKit backend-capacity rejection into a
provider-neutral application error. When multiple backends are active,
operators would benefit from knowing which backend rejected admission without
parsing an error string or exposing endpoint details.
### Current integration
PromptKit provides the useful `ErrCapacityExceeded` sentinel. Notarius can
classify the failure reliably, but it retains only a sanitized diagnostic
string as additional context.
### Requested capability
Add a typed error that continues to match `ErrCapacityExceeded`:
```go
type CapacityError struct {
BackendID string
}
func (e *CapacityError) Is(target error) bool {
return target == ErrCapacityExceeded
}
```
The exact implementation may use `Unwrap` or another idiomatic mechanism. The
important properties are compatibility with `errors.Is` and discoverability
through `errors.As`.
### Design considerations
- Include the stable backend ID.
- Do not expose the backend endpoint, credential environment, credential
value, request content, or other sensitive configuration.
- Consider including the configured concurrency and queue limits if they are
useful and safe, but backend identity alone provides most of the downstream
value.
- Add a retry delay only if PromptKit can provide a meaningful value. A full
queue does not necessarily imply a reliable `Retry-After` duration.
- Keep retry and backoff policy with the consuming application. PromptKit
should classify the admission failure rather than silently retry it.
### Value to Notarius
This would improve operational diagnostics and future metrics while preserving
the provider-neutral error boundary used by Notarius.
## Capabilities PromptKit Already Provides Well
The current PromptKit boundary is sufficient for Notarius's implemented
behavior. In particular, PromptKit already provides:
- filesystem, `fs.FS`, and programmatic prompt, profile, and schema sources;
- offline preparation without model execution;
- structured output and content validation;
- direct session propagation;
- tri-state per-run reasoning overrides;
- selected profile, backend, model, endpoint, effective parameters, hashes, and
token-usage provenance;
- endpoint-only profiles;
- the conventional `local` backend helper;
- arbitrary engine-scoped `Backend` registrations;
- backend authentication environment names, extra parameters, concurrency
limits, and queue-capacity policies;
- provider-client and artifact-reader extension interfaces;
- context cancellation; and
- useful public error sentinels, including profile absence and capacity
exhaustion.
The wishlist does not imply that Notarius needs PromptKit to broaden its core
responsibilities. It primarily asks for more direct access to information and
operations that PromptKit already computes internally.
## Responsibilities That Should Remain In Notarius
The following concerns belong to the downstream application and should not
move into PromptKit for the sake of Notarius:
- pipeline staging, dependencies, and generated references;
- application-wide scheduling across providers and backends;
- module and validation retry policy;
- checkpoints, resume, and recomputation;
- durable run artifacts and manifests;
- D&D prompts, schemas, extractors, validators, and normalizers;
- Notarius configuration-file parsing and precedence;
- domain-specific prompt-cache prefix policy; and
- application-specific redaction, retention, and debug-bundle policy.
PromptKit's complete `Backend` API already supports custom IDs, multiple local
endpoints, authentication, extra parameters, and explicit queue policies.
Whether Notarius exposes those capabilities in its own configuration is an
application-policy decision, not an upstream PromptKit gap.
## Suggested Upstream Sequence
If the PromptKit team chooses to pursue these ideas, the most useful order for
Notarius would be:
1. Add atomic execution that returns prepared details and the completed result.
2. Add prompt-independent profile inspection.
3. Add a semantic execution-target digest, preferably as part of profile
inspection.
4. Add a typed capacity error carrying backend identity.
The first two address concrete workarounds in current Notarius code. The third
would improve checkpoint correctness and reduce coupling. The fourth is
operational polish.

View File

@@ -1,280 +0,0 @@
# Executable Preparation Handles
**Status:** Complete.
## Purpose
Allow a consumer to prepare one exact Promptkit execution, inspect and retain
its credential-redacted public preparation details, and later execute that
already-prepared work without resolving or rendering the request again.
This provides a supported preflight-before-generation boundary for
[Weatherreporter](weatherreporter-promptkit-wishlist.md#priority-1-executable-preparation-handles)
and removes the duplicate `Prepare`-then-`Run` workaround described by
[Notarius](notarius-promptkit-wishlist.md#priority-1-atomic-execution-with-prepared-details).
## Motivation
`Engine.Prepare` currently returns the provenance and rendered details that
consumers need for debugging, persistence, and preflight checks. `Engine.Run`
then performs its own preparation before generation. A consumer that needs
both values must therefore prepare the same logical request twice.
That workaround duplicates source loading, input hashing, schema work, and
template rendering. It also permits filesystem-backed prompts, profiles,
schemas, or inputs to change between the public preparation and the
preparation that actually produces the result.
Promptkit already owns the complete preparation and execution pipeline. An
opt-in prepared-execution handle should expose the missing boundary without
putting rendered content into every `RunResult` or moving persistence policy
into the library.
## Consumer Workflow
The target public workflow is:
```go
prepared, err := engine.PrepareExecution(ctx, request)
if err != nil {
// Handle preparation failure.
return
}
defer prepared.Discard()
details := prepared.Details()
// Persist or inspect a consumer-selected safe subset of details.
result, err := engine.RunPrepared(ctx, prepared)
```
The target public surface is:
```go
type PreparedExecution struct {
// Opaque Promptkit-owned state.
}
func (e *Engine) PrepareExecution(
ctx context.Context,
req RunRequest,
) (*PreparedExecution, error)
func (p *PreparedExecution) Details() PreparedRun
func (p *PreparedExecution) Discard()
func (e *Engine) RunPrepared(
ctx context.Context,
prepared *PreparedExecution,
) (*RunResult, error)
```
The declarations and GoDoc will own the exact implemented contract. The
important public shape is an opaque handle, caller-owned `PreparedRun`
details, an explicit discard operation, and execution through the engine that
created the handle.
## Prepared Snapshot
`PrepareExecution` performs complete preparation without model generation or
backend-capacity admission. It applies the same request validation, source
precedence, backend and profile resolution, credential requirement checks,
output-contract resolution, artifact loading, hashing, schema loading,
session resolution, and rendering behavior as `Prepare`.
A successful handle freezes all source-derived state required for later
execution, including:
- the selected prompt definition, profile, and backend identity;
- the complete effective execution target and request-field presence;
- rendered messages and effective session ID;
- prompt, rendered-prompt, and input hashes;
- the effective output contract and provider-facing structured-output
constraint; and
- private validation state sufficient to validate generated output without
reopening schema files or `fs.FS` resources, including required schema
references.
After `PrepareExecution` succeeds, changes to prompt, profile, schema, input,
or request-owned data cannot change what `RunPrepared` sends to the model or
how it validates the generated output.
The handle retains an internal snapshot independent from values returned by
`Details`. Mutating a returned `PreparedRun`, its maps, slices, messages, or
schema values does not affect later execution. Each `Details` call returns a
fresh caller-owned copy under the existing `PreparedRun` ownership and stable
JSON rules.
## Handle Lifecycle
A `PreparedExecution` is:
- created only by a successful `PrepareExecution` call;
- bound to the exact `Engine` that created it;
- valid for one `RunPrepared` invocation;
- safe for repeated `Details` calls;
- intentionally opaque and without a supported JSON representation; and
- in-process state rather than a durable or restartable job.
`RunPrepared` atomically claims a valid handle before beginning the execution
attempt. A second or concurrent invocation fails without starting another
execution, including when the first invocation ended in cancellation, capacity
rejection, generation failure, or operational validation failure. Copies of
the public handle share the same one-attempt state and cannot bypass this rule.
A nil, zero-value, foreign-engine, discarded, already-claimed, or already-used
handle is invalid. `RunPrepared` reports these lifecycle errors through the
ordinary public invalid-request category. A failed foreign-engine invocation
does not consume a handle that remains valid for its owning engine.
`Discard` idempotently makes an unclaimed handle unavailable for execution and
drops Promptkit's references to secret-bearing or execution-only state.
`RunPrepared` performs the same cleanup automatically after claiming a handle.
Credential-redacted public preparation details remain available after discard,
success, or failure so consumers can retain diagnostics. Promptkit does not
promise secure erasure of Go string memory.
Lifecycle transitions are concurrency-safe. When `RunPrepared` and `Discard`
race, exactly one claims the ready handle. `Discard` is not an execution
cancellation mechanism and does not interrupt an attempt that has already
claimed the handle; consumers cancel that attempt through its context.
## Credentials And Sensitive Data
`Details` has the same security contract as `PreparedRun`: it can contain
rendered messages, schemas, identifiers, and hashes, but never a resolved API
key value. Consumers remain responsible for selecting, redacting, storing, and
retaining any persisted preparation material.
A direct `RunRequest.APIKey` is retained only in opaque execution state until
the handle is run or discarded. It is never added to details, hashes, JSON,
`String`, or `GoString` output.
An environment-variable name is frozen as part of the effective target, but
its credential value is not captured for the lifetime of the handle.
`PrepareExecution` applies the existing preparation-time availability check.
`RunPrepared` rechecks availability before admission, and the selected model
client uses the environment value visible during execution. This preserves
current secret ownership and avoids retaining an environment credential while
a consumer persists preflight material.
The opaque handle must not expose retained request data or credentials through
default formatting, JSON, or error messages.
## Admission, Cancellation, And Execution
`PrepareExecution` never reserves backend admission or an active-generation
permit. Its context governs preparation only; cancellation after it returns
does not invalidate the handle.
`RunPrepared` uses its own context for credential revalidation, backend
admission, active-generation waiting, model generation, output validation, and
any internal repair. Admission occurs when `RunPrepared` begins so a consumer
cannot occupy bounded capacity while inspecting or persisting preparation
details.
For a limited backend, the admission lease covers the complete prepared
execution attempt after admission: generation, validation, internal repair,
and every success or failure exit. Actual generation continues to use the
backend's FIFO active-generation permit. Existing capacity error identity,
cancellation behavior, and release guarantees remain in force.
Because a prepared handle is one-attempt, cancellation or capacity rejection
does not make it reusable. Retry and backoff policy remains with the consumer,
which may create a new prepared handle when another attempt is appropriate.
## Results And Failures
On success, `RunPrepared` returns the existing caller-owned `RunResult`. Its
source-derived provenance must match `Details`, including prompt identity and
hash, rendered-prompt hash, session ID, selected profile and backend,
effective target, and input hashes.
`RunResult.StartTime`, `EndTime`, and `Duration` describe the
`RunPrepared` execution attempt. They exclude preparation time and any delay
while the consumer retained the handle. Preparation timing remains in
`PreparedRun`.
Preparation failure returns no handle. After successful preparation,
`RunPrepared` retains the existing rule that an operational failure returns no
partial `RunResult`; the consumer already has independent preparation details.
A completed content-validation failure remains a successful result with
`ValidationFailed`.
`PrepareExecution` preserves the public error categories of `Prepare`.
`RunPrepared` preserves applicable invalid-request, credential, capacity,
generation, validation, collaborator, and cancellation identities without
reintroducing source-loading or rendering failures from frozen state.
## Compatibility And Existing Workflows
This feature is additive:
- `Prepare` remains the simple preparation-only operation;
- `Run` remains the simple prepare-and-execute operation with its current
early-admission and error-ordering behavior;
- `PreparedRun` and `RunResult` retain their existing stable JSON
representations;
- model-client and artifact-reader extension interfaces remain unchanged; and
- backend routing, concurrency limits, queue capacities, and provider wire
behavior remain unchanged.
The new workflow may share internal machinery with `Prepare` and `Run`, but it
must not change their observable behavior merely to simplify implementation.
## Documentation
The completed documentation set has these ownership boundaries:
- exported declarations and GoDoc own the exact handle, method, lifecycle,
ownership, concurrency, credential, error, and cancellation contracts;
- the promptkit consumer guide explains when to use `Prepare`, `Run`, or the
two-phase prepared-execution workflow; and
- internal runner, source-validation, capacity, and model-client documentation
describe the implemented collaborator boundaries without duplicating public
contracts.
No release document is part of the feature implementation itself. Release
guidance is prepared only when the resulting public API is selected for
publication.
## Non-Goals
This work does not include:
- serializable, durable, resumable, or cross-process execution handles;
- reuse for multiple consumer-initiated executions;
- concurrent execution of one handle;
- capacity reservation during preparation;
- a background task queue, priorities, worker lifecycle, or job status;
- retry, backoff, or provider failover policy;
- a separate `RunDetailed` convenience method;
- adding preparation details or rendered messages to every `RunResult`;
- prompt or profile inspection APIs;
- structured capacity or generation errors;
- freezing environment-variable credential values for the handle lifetime;
- snapshotting provider state or mutable behavior inside an injected
`LLMClient`;
- allowing target, validation, session, variable, or input overrides after
preparation; or
- changing existing `Prepare`, `Run`, file-format, provider-request, or stable
JSON contracts.
## Target End State
After this work:
- consumers can perform and persist preflight before starting provider work;
- one prepared handle executes exactly the source-derived prompt, target,
schema, inputs, session, and messages described by its public details;
- execution never reloads or rerenders consumer sources;
- direct credentials remain confined to opaque, explicitly discardable state;
- environment credentials are not retained across the preflight boundary;
- backend capacity is reserved only when execution begins;
- one handle can start at most one execution attempt, including any internal
repair calls owned by that attempt;
- preparation details remain available after execution success or failure;
- existing simple `Prepare` and `Run` consumers remain unaffected; and
- Promptkit continues to own reusable execution mechanics without taking on
downstream persistence, redaction, retry, or job-management policy.

View File

@@ -1,246 +0,0 @@
# Prompt-Independent Profile Inspection
**Status:** Accepted.
## Purpose
Allow consumers to look up one execution profile by ID and inspect its
structurally resolved model target without selecting a prompt, supplying
placeholder inputs, checking credential availability, or invoking a model.
This provides a direct configuration-validation boundary for
[Notarius](notarius-promptkit-wishlist.md#priority-2-prompt-independent-profile-inspection)
and
[Weatherreporter](weatherreporter-promptkit-wishlist.md#priority-3-prompt-independent-profile-inspection).
## Motivation
Both downstream consumers need to reject invalid configured profile IDs before
starting application work. They currently have to construct a synthetic prompt
and call `Engine.Prepare` merely to exercise profile loading, backend lookup,
and execution-target resolution.
That workaround couples profile validation to unrelated prompt definitions,
fixture inputs, rendering, schema behavior, and current credential
availability. Promptkit already owns profile precedence, backend membership,
and target resolution, so it should expose that cohesive capability directly.
## Consumer Workflow
The target public workflow is:
```go
inspection, err := engine.InspectProfile(ctx, profileID)
if err != nil {
// Reject or report the configured profile.
return
}
target := inspection.EffectiveModelParams
if target.APIKeyEnv != "" {
// Apply application policy for the named environment variable.
}
```
The target public surface is:
```go
type ProfileInspection struct {
ProfileID string
EffectiveModelParams ExecutionTarget
APIKeyRequired bool
}
func (e *Engine) InspectProfile(
ctx context.Context,
profileID string,
) (*ProfileInspection, error)
```
The declarations and GoDoc will own the exact implemented contract. The
important public shape is exact lookup through the existing engine, one
caller-owned inspection value, the resolved `ExecutionTarget`, and an explicit
signal for a direct API-key requirement.
`EffectiveModelParams.BackendID` identifies the selected registered backend
and remains empty for endpoint-only profiles.
`EffectiveModelParams.APIKeyEnv` reports the effective credential environment
variable name. `APIKeyRequired` reports that the profile requires a direct
request credential instead. These states are mutually exclusive after normal
profile and backend precedence is applied.
`ProfileInspection` does not need a stable JSON representation. Consumers that
persist application configuration or diagnostics can select the fields their
own format requires.
## Lookup And Precedence
`InspectProfile` requires a non-blank explicit profile ID. It trims surrounding
whitespace and otherwise performs the same case-sensitive exact lookup used by
ordinary execution.
Lookup applies the engine's normal profile-source precedence:
- programmatic profiles supplied through `WithProfiles`;
- the configured file, directory, or `fs.FS` profile source; and
- the built-in profile catalog.
A valid higher-precedence match shadows a lower-precedence profile with the
same ID. A malformed or unreadable higher-precedence match fails rather than
silently falling back. The
[profile format reference](../formats.md#profile-definitions) remains the
canonical owner of profile-source and file-format behavior.
Inspection never derives a profile ID from a prompt's `default_profile`; the
caller is inspecting one explicitly named profile.
## Structural Resolution
Inspection loads and validates the selected profile, verifies that a named
backend exists in the engine's immutable backend registry, and applies normal
framework-default, backend, and profile precedence to produce the effective
target.
For the same engine state and profile ID, with no per-run execution override,
the inspected target must match the target that ordinary preparation would
resolve before applying request credentials and checking their availability.
This equivalence must use one shared resolution path rather than a second set
of precedence rules.
Structural resolution includes:
- backend routing identity;
- endpoint and model;
- sampling, token, timeout, service-tier, and reasoning settings;
- the effective credential environment-variable name or direct-key
requirement; and
- deeply copied provider-specific extra parameters.
Inspection returns the effective target rather than a raw profile definition.
This keeps framework and backend defaults visible to consumers without
creating a second public profile-loading interface.
The result does not include request-override presence because no
`ExecutionTargetOverride` participates in inspection.
## Credentials And Sensitive Data
Inspection reports credential requirements but never resolves, retains, or
returns a credential value.
The operation does not read the named environment variable and succeeds when
that variable is absent or blank. It accepts neither a direct API key nor an
API-key environment override. Consumers decide whether credential availability
must be enforced during application configuration, while `Prepare`,
`PrepareExecution`, `Run`, and `RunPrepared` retain their execution-time
credential contracts.
Error messages, formatting, and returned values must not expose environment
values or other resolved secrets. Existing restrictions against raw API keys
in profile sources remain unchanged.
## Ownership, Consistency, And Concurrency
Each successful call returns a caller-owned snapshot. Mutating the returned
target or any nested extra-parameter map or slice cannot affect the engine,
later inspection, or later execution.
Inspection is safe to call concurrently under the engine's existing immutable
registry and repository contracts. It does not mutate profile sources or
cache a result globally.
For filesystem-backed sources, an inspection describes the state observed by
that call. It does not freeze the profile for a later `Run`; a source may
change between operations. Consumers requiring an exact preflight-to-execution
snapshot should use the existing prepared-execution workflow.
## Errors And Cancellation
The operation uses existing public error categories:
- a blank profile ID matches `ErrInvalidRequest`;
- an absent exact ID matches `ErrProfileNotFound` and not `ErrProfileLoad`;
- read, decode, validation, and source-selection failures match
`ErrProfileLoad`; and
- an unknown referenced backend or an invalid structurally resolved target
matches `ErrProfileLoad`.
Errors should preserve useful underlying collaborator and context identities
through `errors.Is` where the existing facade does so, without exposing
internal package types. Context cancellation governs inspection and no partial
inspection result is returned on failure.
Missing credential values are not inspection errors. The operation cannot
return capacity or model-generation failures because it performs neither
backend admission nor generation.
## Compatibility And Boundaries
This feature is additive. Existing profile formats, source precedence,
backend registration, `Prepare`, prepared execution, and `Run` behavior remain
unchanged.
The method belongs on the root `Engine` facade. Profile repositories and the
backend registry remain internal implementation details, and no new public
repository interface is introduced.
Inspection does not require prompt lookup, rendering, artifact loading, schema
loading, validation, backend-capacity admission, or model-client access.
Engine construction retains its ordinary configuration requirements; this
feature does not introduce a separate profile-only engine.
## Documentation
The completed documentation set has these ownership boundaries:
- exported declarations and GoDoc own the exact method, result, ownership,
credential, error, and cancellation contracts;
- the promptkit consumer guide explains configuration-time profile inspection
and distinguishes it from `Prepare` and prepared execution;
- the profile format reference continues to own profile fields and source
precedence; and
- internal documentation describes shared profile and target resolution
without duplicating public contracts.
## Non-Goals
This work does not include:
- enumerating or searching profiles;
- returning raw profile definitions or profile source paths;
- accepting per-run execution overrides, direct API keys, or API-key
environment overrides;
- checking environment-variable contents or other credential availability;
- semantic execution-target fingerprints or profile hashes;
- prompt-definition inspection or prompt default-profile resolution;
- full-corpus validation across every profile source;
- freezing a filesystem-backed profile for later execution;
- exposing backend concurrency limits, queue state, or capacity policy;
- model generation, provider health checks, or endpoint connectivity tests;
- dynamic backend or profile registration after engine construction; or
- changing current profile, backend, prepared-execution, or stable JSON
contracts.
## Target End State
After this work:
- consumers can validate one configured profile without inventing a prompt or
placeholder inputs;
- lookup observes ordinary programmatic, configured-source, and built-in
precedence;
- a successful result proves that the profile exists, is valid, references a
registered backend when applicable, and resolves to a structurally valid
effective target;
- the inspected target matches ordinary preparation for the same profile and
engine state before per-run overrides and credential availability checks;
- credential requirements are visible without reading or exposing credential
values;
- returned targets and nested data are caller-owned;
- inspection performs no rendering, source loading unrelated to the profile,
capacity admission, or model work;
- existing execution workflows and compatibility contracts remain unchanged;
and
- Promptkit owns reusable profile validation while downstream applications
retain configuration policy, persistence, logging, and credential-timing
decisions.

View File

@@ -1,468 +0,0 @@
# Weatherreporter PromptKit Wishlist
## Purpose
This document records features and interface changes that would be useful
additions to PromptKit from the perspective of the maintainers of
Weatherreporter, a downstream application planning to replace its Scriptorium
CLI integration with PromptKit.
PromptKit v0.3.0 provides the capabilities Weatherreporter needs for the
migration. None of the ideas below is a hard adoption requirement. They are
opportunities to avoid duplicate preparation, validate configuration earlier,
improve durable failure diagnostics, and make the integration more direct.
The examples are API sketches intended to communicate the desired capability,
not prescriptive names or finalized Go contracts. The related
[Notarius PromptKit wishlist](notarius-promptkit-wishlist.md) proposes several
overlapping features from another downstream consumer's perspective.
## Priority 1: Executable Preparation Handles
**Disposition:** Accepted into the
[executable preparation handles](prepared-execution.md) feature roadmap.
### Downstream need
Weatherreporter treats prompt preparation as a durable preflight boundary. It
needs to:
1. prepare the exact request that will be executed;
2. persist a safe preparation record before starting the provider call; and
3. execute without reloading or rerendering prompt, profile, schema, or input
sources.
Persisting preflight before generation leaves useful evidence when a provider
call fails or the process is interrupted during generation.
### Current integration option
With PromptKit v0.3.0, Weatherreporter can call `Engine.Prepare`, save selected
fields from the returned `PreparedRun`, and then call `Engine.Run` with the
same request. Because `Run` performs preparation internally, the work is
repeated.
Weatherreporter plans to use embedded prompt and schema files plus immutable
inline input bytes, which removes most of the consistency risk. An external
profile file or directory can still change between the two calls, and the
second preparation remains unnecessary work.
The atomic `RunDetailed` operation proposed by the
[Notarius wishlist](notarius-promptkit-wishlist.md#priority-1-atomic-execution-with-prepared-details)
would guarantee that returned preparation details describe the completed
execution. However, returning those details only after generation would not
preserve Weatherreporter's preflight-before-generation persistence boundary.
### Requested capability
Add an opt-in two-phase API that returns a prepared execution handle:
```go
prepared, err := engine.PrepareExecution(ctx, request)
if err != nil {
// Handle preparation failure.
}
details := prepared.Details()
// Persist a consumer-selected safe preparation record.
result, err := engine.RunPrepared(ctx, prepared)
```
The exact names and shapes are flexible. The important contract is that
`RunPrepared` executes the already prepared prompt and does not reload or
rerender its prompt, profile, schema, or input sources.
`Details` should return the same caller-owned public preparation information
currently represented by `PreparedRun`. The execution handle may retain opaque
engine-owned state needed to invoke the model and validate the response.
### Design considerations
- Keep `Prepare` and `Run` available for consumers that do not need a
two-phase execution boundary.
- Bind a prepared handle to the engine that constructed it.
- Define whether a handle is one-shot, reusable, or safe for concurrent use.
A one-shot contract may be the safest initial design.
- Do not give the opaque handle a stable JSON representation.
- Do not expose or serialize resolved credential values through `Details`.
- Define how a direct request API key is retained and released when an opaque
handle must carry it until execution.
- Preserve caller-owned copies for all public details.
- Make context cancellation and backend admission timing explicit.
- Document whether profile credential environment values are resolved during
preparation or execution.
- Ensure an execution error does not invalidate the public details already
returned to the consumer.
- Consider whether an atomic `RunDetailed` can share the same internal
prepared-execution implementation.
### Value to Weatherreporter
This is the highest-value upstream addition. It would preserve
Weatherreporter's durable preflight behavior, remove duplicate work, eliminate
the remaining source-consistency window, and ensure that persisted provenance
describes the actual execution.
## Priority 2: Prompt-Definition Inspection
**Disposition:** Accepted into the
[future catalog](future.md#prompt-definition-inspection).
### Downstream need
Weatherreporter has a fixed registry of seven report definitions. Each report
selects a prompt ID and one of two output workflows:
- direct Markdown; or
- structured generated text followed by application-owned domain validation
and Markdown template rendering.
Weatherreporter will embed the PromptKit prompt definitions and private
response schemas that implement those reports. It needs to validate that the
report registry and embedded prompt corpus agree before weather collection or
provider execution.
### Current integration option
Weatherreporter can maintain synthetic data-package fixtures and call
`Engine.Prepare` for every report prompt during tests. Runtime validation can
also occur through the ordinary per-report preparation stage.
This works, but it requires complete placeholder inputs and profile resolution
when the application primarily wants to inspect prompt identity and declared
contracts.
### Requested capability
Add exact prompt-definition lookup without rendering or generation:
```go
type PromptInfo struct {
PromptID string
PromptVersion string
PromptHash string
DefaultProfileID string
Inputs []InputDefinition
OutputContract OutputContract
}
func (e *Engine) ResolvePrompt(
ctx context.Context,
promptID string,
promptVersion string,
) (PromptInfo, error)
```
The exact returned shape may differ. Weatherreporter needs enough information
to verify prompt existence, version selection, declared inputs, default
profile identity, output format, validation mode, and schema selection without
supplying synthetic prompt input.
### Design considerations
- Use ordinary PromptKit prompt-source precedence and exact ID/version
selection.
- Fully load and structurally validate the selected prompt definition.
- Validate referenced prompt content files without rendering their templates.
- Resolve and validate the selected output contract and schema reference where
practical.
- Return an opaque prompt-definition equality value rather than raw source
bytes.
- Do not return rendered messages, schema bodies, profile credentials, or
another source of sensitive content.
- Preserve typed or sentinel errors for missing and invalid prompts.
- Return caller-owned values.
- Enumeration of all known prompts is not required for Weatherreporter; exact
lookup is sufficient.
### Value to Weatherreporter
This would let Weatherreporter directly verify that every report prompt
exists, requires the curated `data_package` input, and declares the expected
Markdown or JSON Schema output contract. It would reduce synthetic test setup
and move failures ahead of weather collection.
## Priority 3: Prompt-Independent Profile Inspection
**Disposition:** Covered by the accepted
[prompt-independent profile inspection](profile-inspection.md) roadmap.
### Downstream need
Weatherreporter will allow operators to select an external PromptKit profile
source and may allow an explicit profile override. It should reject a missing
profile, unknown backend, malformed execution target, or unsatisfied credential
requirement before collecting weather data or writing report artifacts.
### Current integration option
Weatherreporter can validate an explicit profile by preparing one embedded
prompt with fixture input. Prompts that use their own default profiles can be
validated during their normal preparation stage.
This couples configuration validation to one prompt and requires placeholder
input even when only profile and backend resolution are relevant.
### Requested capability
The prompt-independent `ResolveProfile` API proposed by the
[Notarius wishlist](notarius-promptkit-wishlist.md#priority-2-prompt-independent-profile-inspection)
would satisfy this need. It should resolve built-in, file-backed, and
programmatic profiles, validate backend membership, report credential
requirements without resolving credential values, and preserve typed error
classification.
### Additional Weatherreporter considerations
- An explicit application profile override should be inspectable without
selecting a report prompt.
- A prompt-definition inspection result may expose its default profile ID so
Weatherreporter can inspect that profile separately.
- Inspection should distinguish structural profile validity from current
credential availability so configuration validation can apply explicit
application policy.
- An optional execution-target override should be considered only if it
describes the same target that a later run will use.
### Value to Weatherreporter
This would improve fail-fast configuration validation and give operator-facing
errors direct profile and backend context. It is valuable but not required for
the initial migration.
## Priority 4: Eager Source Validation
**Disposition:** Deferred until prompt and profile inspection have been used
to determine whether a broader engine-wide validation operation is still
needed.
### Downstream need
PromptKit deliberately defers reading and validating filesystem and `fs.FS`
prompt, profile, and schema content until a request needs it. Weatherreporter
has a small fixed embedded prompt corpus and one optional external profile
source. It would benefit from an explicit offline validation operation for
tests, startup diagnostics, and configuration checks.
### Current integration option
Weatherreporter can prepare every report prompt with fixture inputs and inspect
any explicit profiles individually. That provides strong coverage but requires
consumer-maintained traversal and synthetic material.
### Requested capability
Consider an opt-in source-validation operation:
```go
type SourceValidationOptions struct {
RequireCredentials bool
}
func (e *Engine) ValidateSources(
ctx context.Context,
opts SourceValidationOptions,
) error
```
The operation should eagerly discover and structurally validate the configured
prompt, profile, and schema sources without model generation.
### Design considerations
- Keep deferred validation as the normal `NewEngine` behavior.
- Make eager validation an explicit consumer choice.
- Validate duplicate IDs and versions, strict YAML decoding, referenced content
files, profile/backend membership, schema syntax, and schema references.
- Distinguish structural credential declarations from current environment
availability.
- Do not read or expose credential values when credential availability is not
requested.
- Preserve source-specific public error identities and useful path context.
- Respect context cancellation during filesystem discovery and schema work.
- Consider whether exact prompt and profile inspection APIs already provide a
smaller sufficient surface before adding an engine-wide operation.
### Value to Weatherreporter
This would simplify offline corpus checks and catch malformed operator profile
sources before report work begins. It is helpful but lower priority than exact
prompt and profile inspection.
## Priority 5: Structured Generation Errors
**Disposition:** Deferred pending stronger downstream demand and a narrower
design that does not duplicate prepared provenance or impose HTTP-specific
fields on injected model clients.
### Downstream need
Weatherreporter preserves redacted, inspectable failure receipts for report
runs. When model generation fails operationally, it needs to classify the
failure and retain safe execution context without parsing error prose.
Prompt preparation already supplies selected profile, backend, and model
identity. Provider status classification would add useful operator context,
especially when the built-in OpenAI-compatible client receives a non-success
HTTP status.
### Current integration option
PromptKit exposes `ErrLLMGenerate` and preserves injected client errors through
`errors.Is`. Weatherreporter can reliably classify generation failure and use
its preparation record for profile, backend, and model provenance. Any further
diagnostic detail remains a redacted error string.
### Requested capability
Consider a typed generation error that continues to match `ErrLLMGenerate`:
```go
type GenerationError struct {
BackendID string
Model string
StatusCode int
}
```
The exact fields may differ. The useful contract is safe structured context
available through `errors.As`, while `errors.Is(err, ErrLLMGenerate)` remains
compatible.
### Design considerations
- Include only fields that PromptKit knows reliably and can expose safely.
- Treat an HTTP status as optional because injected model clients may not use
HTTP.
- Do not expose provider response bodies, endpoints, credential environment
names, credential values, request content, or generated content.
- Do not make a structured error a second source of prompt/profile provenance
already present in a prepared execution.
- Preserve injected client error identity.
- Keep retry and backoff policy with the consuming application.
### Value to Weatherreporter
This would improve durable failure receipts and troubleshooting, particularly
for built-in transport failures. It is not required if preparation details and
the existing sentinel remain available.
## Lower-Priority Shared Wishlist Items
### Structured Capacity Errors
**Disposition:** Accepted into the
[future catalog](future.md#structured-capacity-errors).
The typed capacity error proposed by the
[Notarius wishlist](notarius-promptkit-wishlist.md#priority-4-structured-capacity-errors)
would improve Weatherreporter diagnostics by exposing the stable backend ID
without parsing error text.
Weatherreporter currently generates batch reports sequentially and constructs
one engine per invocation, so engine-local capacity exhaustion is unlikely in
the initial design. The feature would become more valuable if report
generation later becomes concurrent or PromptKit engines become longer-lived.
It should not block adoption.
### Semantic Execution-Target Fingerprints
**Disposition:** Deferred until prompt-independent profile inspection defines
the resolved target whose configuration identity would be fingerprinted.
The semantic target digest proposed by the
[Notarius wishlist](notarius-promptkit-wishlist.md#priority-3-semantic-execution-target-fingerprints)
would provide a compact equality signal for audit metadata.
Weatherreporter does not currently reuse LLM-dependent checkpoints. Its Recent
Changes behavior compares deterministic module snapshots rather than generated
reports, so the digest has no immediate cache-correctness role. Existing
PromptKit result metadata is sufficient for the initial integration. A digest
would still be useful provenance and future-proofing, but it is not a
migration priority.
## Capabilities PromptKit Already Provides Well
PromptKit v0.3.0 already provides the essential Weatherreporter integration
surface:
- importable in-process engine construction;
- filesystem, `fs.FS`, and programmatic prompt, profile, and schema sources;
- offline preparation without model execution;
- versioned prompt selection;
- text, Markdown, JSON, and JSON Schema output contracts;
- single-pass output validation with raw output retained after completed
validation failure;
- inline input artifacts with provenance URIs and input hashes;
- selected profile, backend, model, effective target, prompt hashes, timing,
and token-usage provenance;
- endpoint-only profiles and engine-scoped backend registration;
- injected model-client and artifact-reader interfaces;
- caller cancellation, generation timeout, and transport timeout behavior; and
- public error sentinels for configuration, prompt, profile, artifact,
validation, capacity, and generation failures.
These capabilities are sufficient for Weatherreporter to adopt PromptKit
without waiting for new upstream work.
## Responsibilities That Should Remain In Weatherreporter
The following concerns belong to Weatherreporter and should not move into
PromptKit:
- report definitions, valid periods, batches, and output naming;
- application prompt content and private report response schemas;
- deterministic weather facts, modules, and Recent Changes;
- curated `data_package` construction and persistence;
- generated-text domain validation and Markdown template rendering;
- managed artifact paths, atomic writes, metadata, and inspection commands;
- preparation, execution, raw-output, and failure-receipt schemas;
- CLI configuration loading and precedence;
- debug enablement, redaction, placement, sensitivity, and retention;
- distributor notification;
- batch continuation and any future retry policy; and
- application-level compatibility and migration policy.
## Suggested Upstream Sequence
If the PromptKit team chooses to pursue these ideas, the most useful order for
Weatherreporter would be:
1. Add executable preparation handles, ideally sharing implementation with an
atomic detailed-run API.
2. Add prompt-definition inspection.
3. Add prompt-independent profile inspection.
4. Consider eager source validation after evaluating whether the two exact
inspection APIs are sufficient.
5. Add structured generation errors.
6. Add structured capacity errors and semantic execution-target fingerprints
as lower-priority operational improvements.
The first item removes the only material integration workaround. Prompt and
profile inspection improve fail-fast validation. The remaining items improve
ergonomics and diagnostics.
## Adoption Sequencing
Weatherreporter should not wait for the complete wishlist. PromptKit v0.3.0 is
already sufficient when Weatherreporter:
- embeds immutable prompt and schema assets;
- supplies immutable inline data-package bytes;
- constructs one engine per CLI invocation;
- calls `Prepare` and `Run` with the same request; and
- keeps PromptKit behind a weatherreporter-owned adapter contract.
If executable preparation handles are scheduled for a near-term PromptKit
release, Weatherreporter may defer only its final adapter implementation to
avoid implementing and then removing duplicate preparation. Prompt corpus
retrieval, application-contract design, configuration work, embedded assets,
state contracts, and offline fixtures can proceed independently.
If the feature is not scheduled, Weatherreporter can adopt v0.3.0 and keep the
duplicate `Prepare` and `Run` sequence inside its adapter. A later PromptKit
upgrade would remain localized behind that neutral boundary.
Prompt inspection, profile inspection, source validation, structured errors,
capacity details, and semantic fingerprints should not gate adoption.

252
engine.go
View File

@@ -65,8 +65,8 @@ var (
ErrPromptRender = errors.New("failed to render prompt")
// ErrCapacityExceeded identifies a Run or RunPrepared rejected because the
// selected backend already admitted ConcurrencyLimit + QueueCapacity calls.
// It is not an invalid request, an LLM or provider rate-limit response, or
// ErrLLMGenerate.
// A [CapacityError] reports the selected backend ID. It is not an invalid
// request, an LLM or provider rate-limit response, or ErrLLMGenerate.
ErrCapacityExceeded = errors.New("backend capacity exceeded")
// ErrLLMGenerate identifies a model-client failure or a nil successful
// response. Errors returned by an injected LLMClient remain available
@@ -78,14 +78,15 @@ var (
ErrValidation = errors.New("failed to validate output")
)
// Engine prepares and runs Promptkit prompt requests.
// Engine inspects prompts and profiles and prepares and runs Promptkit prompt
// requests.
//
// An Engine is safe for concurrent calls to [Engine.Prepare],
// [Engine.PrepareExecution], [Engine.Run], and [Engine.RunPrepared]. Each
// Engine owns independent backend-capacity pools that coordinate Run and
// RunPrepared admission and model generation. Injected collaborators may still
// be invoked concurrently across different backend pools or for unlimited
// backends.
// An Engine is safe for concurrent calls to [Engine.InspectPrompt],
// [Engine.InspectProfile], [Engine.Prepare], [Engine.PrepareExecution],
// [Engine.Run], and [Engine.RunPrepared]. Each Engine owns independent
// backend-capacity pools that coordinate Run and RunPrepared admission and
// model generation. Injected collaborators may still be invoked concurrently
// across different backend pools or for unlimited backends.
type Engine struct {
runner *usecase.Runner
}
@@ -97,9 +98,10 @@ type Config struct {
// It is required unless a WithPromptFS or WithPromptFile option supplies the
// prompt source.
PromptDir string
// ProfileDir is an optional directory whose profiles take precedence over
// embedded built-in profiles. An empty value selects only built-ins unless
// profile options are also supplied.
// ProfileDir is an optional ordinary configured source whose profiles take
// precedence over application fallback and embedded built-in profiles. An
// empty value selects the lower-precedence sources unless a profile-source
// option supplies the ordinary source.
ProfileDir string
// SchemaDir is the root for JSON Schema files. An empty value uses the
// current directory. WithSchemaFS or WithSchemaFile replaces this source.
@@ -118,12 +120,12 @@ type Config struct {
// Option customizes engine construction.
//
// NewEngine applies options in argument order and ignores nil options. Within
// each prompt-source, profile-source, in-memory-profile, schema-source,
// model-client, and artifact-reader category, the last non-nil valid option
// replaces earlier options in that category. WithBackend is the additive
// exception: unique registrations accumulate, and a repeated backend ID is an
// error rather than a replacement. An invalid option fails construction even
// if a later option would replace it.
// each prompt-source, ordinary-profile-source, fallback-profile-source,
// in-memory-profile, schema-source, model-client, and artifact-reader
// category, the last non-nil valid option replaces earlier options in that
// category. WithBackend is the additive exception: unique registrations
// accumulate, and a repeated backend ID is an error rather than a replacement.
// An invalid option fails construction even if a later option would replace it.
type Option interface {
apply(*engineOptions) error
}
@@ -135,18 +137,20 @@ func (f optionFunc) apply(options *engineOptions) error {
}
type engineOptions struct {
llmClient llm.Client
artifactReader artifactadapter.Reader
promptDefs promptdef.Repository
profiles profile.Repository
memoryProfiles profile.Repository
backends []domain.Backend
validator validate.Validator
promptSource bool
profileSource bool
memorySource bool
validatorSource bool
artifactSource bool
llmClient llm.Client
artifactReader artifactadapter.Reader
promptDefs promptdef.Repository
profiles profile.Repository
fallbackProfiles profile.Repository
memoryProfiles profile.Repository
backends []domain.Backend
validator validate.Validator
promptSource bool
profileSource bool
fallbackProfileSource bool
memorySource bool
validatorSource bool
artifactSource bool
}
// WithLLMClient replaces the built-in model client used by [Engine.Run] and
@@ -223,12 +227,12 @@ func WithPromptFile(path string) Option {
// WithProfileFS loads execution profiles from fsys under root.
//
// Profiles from this source overlay built-in profiles. Profile YAML must use
// api_key_env for environment-based credentials; raw API keys are rejected.
// fsys must be non-nil and root must be non-empty; otherwise NewEngine fails
// with ErrInvalidConfig. This option replaces Config.ProfileDir and earlier
// file or FS profile-source options, but remains below WithProfiles in
// precedence.
// Profiles from this ordinary configured source take precedence over
// application fallback and built-in profiles. Profile YAML must use api_key_env
// for environment-based credentials; raw API keys are rejected. fsys must be
// non-nil and root must be non-empty; otherwise NewEngine fails with
// ErrInvalidConfig. This option replaces Config.ProfileDir and earlier file or
// FS profile-source options, but remains below WithProfiles in precedence.
func WithProfileFS(fsys fs.FS, root string) Option {
return optionFunc(func(options *engineOptions) error {
if fsys == nil {
@@ -245,11 +249,12 @@ func WithProfileFS(fsys fs.FS, root string) Option {
// WithProfileFile loads execution profiles from the single profile file at path.
//
// The profile overlays built-in profiles. Profile YAML must use api_key_env for
// environment-based credentials; raw API keys are rejected. path must name an
// existing non-directory file when NewEngine applies the option. This option
// replaces Config.ProfileDir and earlier file or FS profile-source options,
// but remains below WithProfiles in precedence.
// The profile takes precedence over application fallback and built-in profiles.
// Profile YAML must use api_key_env for environment-based credentials; raw API
// keys are rejected. path must name an existing non-directory file when
// NewEngine applies the option. This option replaces Config.ProfileDir and
// earlier file or FS profile-source options, but remains below WithProfiles in
// precedence.
func WithProfileFile(path string) Option {
return optionFunc(func(options *engineOptions) error {
fsys, root, err := fileSource(path)
@@ -262,8 +267,41 @@ func WithProfileFile(path string) Option {
})
}
// WithFallbackProfileFS supplies application-owned fallback profile
// definitions from fsys under root.
//
// Profile lookup checks, in order, profiles supplied by WithProfiles; the
// ordinary configured source selected by WithProfileFile, WithProfileFS, or
// Config.ProfileDir; this fallback source; and Promptkit's embedded built-in
// profiles. Each source supplies a complete profile definition; profile fields
// are not merged between sources. Only an absent profile ID proceeds to the
// next source. A matching read, parse, duplicate, validation, or credential
// format failure stops resolution.
//
// Files use the ordinary strict profile YAML and api_key_env credential rules.
// Loading and validation are lazy: NewEngine validates this option's arguments
// but does not read profile files. fsys must be non-nil and root must be
// nonblank; otherwise NewEngine returns an error matching ErrInvalidConfig.
// Repeating this option replaces the earlier valid fallback source.
//
// This option controls profile-definition lookup, not provider or generation
// failover.
func WithFallbackProfileFS(fsys fs.FS, root string) Option {
return optionFunc(func(options *engineOptions) error {
if fsys == nil {
return ErrInvalidConfig
}
if strings.TrimSpace(root) == "" {
return ErrInvalidConfig
}
options.fallbackProfiles = profile.NewFSRepository(fsys, root)
options.fallbackProfileSource = true
return nil
})
}
// WithProfiles configures in-memory profiles that take precedence over
// configured profile files and built-in profiles.
// ordinary configured, application fallback, and built-in profiles.
//
// NewEngine validates and copies every profile. IDs must be unique within one
// call. An invalid profile, duplicate ID, or unsupported ExtraParams value
@@ -349,13 +387,7 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
promptDefs = promptdef.NewFilesystemRepository(cfg.PromptDir)
}
profiles := builtin.NewRepositoryWithDirectory(cfg.ProfileDir)
if options.profileSource {
profiles = builtin.NewRepositoryWithPrimary(options.profiles)
}
if options.memorySource {
profiles = profile.NewOverlayRepository(options.memoryProfiles, profiles)
}
profiles := newProfileRepository(cfg.ProfileDir, options)
backendRegistry, err := backend.NewRegistry(options.backends)
if err != nil {
@@ -408,6 +440,26 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
}, nil
}
func newProfileRepository(profileDir string, options engineOptions) profile.Repository {
repository := builtin.NewRepository()
if options.fallbackProfileSource {
repository = profile.NewOverlayRepository(options.fallbackProfiles, repository)
}
if options.profileSource {
repository = profile.NewOverlayRepository(options.profiles, repository)
} else if strings.TrimSpace(profileDir) != "" {
repository = profile.NewOverlayRepository(profile.NewFilesystemRepository(profileDir), repository)
}
if options.memorySource {
repository = profile.NewOverlayRepository(options.memoryProfiles, repository)
}
return repository
}
func fileSource(name string) (fs.FS, string, error) {
cleanName := strings.TrimSpace(name)
if cleanName == "" {
@@ -428,6 +480,93 @@ func fileSource(name string) (fs.FS, string, error) {
return os.DirFS(dir), filepath.ToSlash(base), nil
}
// InspectPrompt resolves one explicit prompt definition without selecting a
// profile or starting execution work.
//
// InspectPrompt requires a nonblank promptID. It passes nonblank promptID and
// promptVersion values unchanged to the engine's ordinary, case-sensitive
// prompt selection. An empty version succeeds only when that source has one
// selected ID; a nonempty version selects one exact ID/version pair. The
// configured prompt source is used without merging, fallback, or enumeration.
//
// A successful result proves that the selected definition and any referenced
// message content files were structurally loaded. Inputs are returned in
// definition order. DefaultProfileID is declared metadata only and is not
// resolved. OutputContract is the normalized declared contract, with a JSON
// Schema path when declared but without loading or compiling that schema.
// PromptHash is the same opaque equality value as PreparedRun.PromptHash for
// the selected definition and observed source state; its spelling, length,
// encoding, algorithm, and security properties are not contracts.
//
// This method does not return prompt bodies, templates, source paths, schemas,
// rendered messages, or execution settings. It does not resolve a profile or
// credential, read artifacts or schemas, render, validate, admit capacity,
// contact a provider, or generate model output. The returned PromptInspection
// and its input slice are caller-owned. Filesystem-backed inspection is a
// point-in-time lookup and does not freeze a definition for later execution.
//
// A nil Engine returns an error matching ErrInvalidConfig. A blank prompt ID
// matches ErrInvalidRequest. An absent exact ID or version matches
// ErrPromptNotFound and not ErrPromptLoad. Malformed, unreadable, duplicate,
// ambiguous, referenced-content, or hashing failures match ErrPromptLoad.
// Cancellation during lookup matches ErrPromptLoad while preserving the
// context error. InspectPrompt returns no partial result on error.
func (e *Engine) InspectPrompt(
ctx context.Context,
promptID string,
promptVersion string,
) (*PromptInspection, error) {
if e == nil || e.runner == nil {
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
}
inspection, err := e.runner.InspectPrompt(ctx, promptID, promptVersion)
if err != nil {
return nil, mapPublicError(err)
}
return fromDomainPromptInspection(inspection), nil
}
// InspectProfile resolves one explicit profile without selecting a prompt or
// starting execution work.
//
// InspectProfile trims surrounding whitespace from profileID and looks up the
// resulting nonblank ID exactly and case-sensitively through the engine's
// in-memory, ordinary configured-source, application fallback, and built-in
// profile precedence. It applies the framework timeout baseline, selected
// backend, and then selected profile to EffectiveModelParams without a request
// override. BackendID is empty for an endpoint-only profile.
//
// APIKeyEnv in the returned target is an environment-variable name, never its
// value. APIKeyRequired instead reports a direct credential requirement and is
// mutually exclusive with a nonblank APIKeyEnv. InspectProfile neither derives
// an ID from a prompt default_profile nor checks credential availability, so an
// absent or blank named environment variable is not an error.
//
// The returned ProfileInspection and all nested mutable values are
// caller-owned. Filesystem-backed inspection is a point-in-time lookup and
// does not freeze the profile for a later execution. This method does not load
// a prompt, render, read artifacts or schemas, admit backend capacity, contact
// a provider, or generate model output.
//
// A nil Engine returns an error matching ErrInvalidConfig. A blank profile ID
// matches ErrInvalidRequest. An absent exact ID matches ErrProfileNotFound and
// not ErrProfileLoad. Malformed or unreadable profile data, an unknown backend,
// or an invalid resolved target matches ErrProfileLoad. Cancellation during
// profile loading matches ErrProfileLoad while preserving the context error.
// InspectProfile returns no partial result on error.
func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*ProfileInspection, error) {
if e == nil || e.runner == nil {
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
}
inspection, err := e.runner.InspectProfile(ctx, profileID)
if err != nil {
return nil, mapPublicError(err)
}
return fromDomainProfileInspection(inspection), nil
}
// Prepare resolves and renders a prompt request without calling an LLM.
//
// Prepare selects the prompt and profile, resolves any selected backend and
@@ -504,9 +643,10 @@ func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*Prepare
// single-pass even when OutputContract.RepairAttempts is positive.
//
// Run can return every error category documented by [Engine.Prepare], plus
// ErrCapacityExceeded and ErrLLMGenerate. ErrCapacityExceeded identifies
// rejection before artifacts, schemas, rendering, or model generation because
// the selected backend's admission capacity is full; it does not match
// ErrCapacityExceeded and ErrLLMGenerate. An engine admission rejection is
// discoverable as [CapacityError] and still matches ErrCapacityExceeded. It
// occurs before artifacts, schemas, rendering, or model generation because the
// selected backend's admission capacity is full; it does not match
// ErrInvalidRequest or ErrLLMGenerate. Errors from injected clients remain
// available through errors.Is. Cancellation while waiting for model-generation
// capacity matches both ErrLLMGenerate and the context error. Cancellation
@@ -547,9 +687,11 @@ func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
//
// RunPrepared can return ErrInvalidRequest, ErrAPIKeyEnvMissing,
// ErrCapacityExceeded, ErrLLMGenerate, or ErrValidation as applicable while
// preserving documented collaborator and context identities. A completed
// content-validation rejection is returned in RunResult, not as an
// operational error. An operational error returns no partial RunResult.
// preserving documented collaborator and context identities. An engine
// admission rejection is discoverable as [CapacityError] and still matches
// ErrCapacityExceeded. A completed content-validation rejection is returned
// in RunResult, not as an operational error. An operational error returns no
// partial RunResult.
func (e *Engine) RunPrepared(ctx context.Context, prepared *PreparedExecution) (*RunResult, error) {
if e == nil || e.runner == nil {
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)

View File

@@ -360,14 +360,14 @@ func TestEngineExecutionSettingPrecedence(t *testing.T) {
wantPresence promptkit.ExecutionTargetPresence
}{
{
name: "framework defaults fill zero-valued profile settings",
name: "unspecified provider controls retain framework timeout",
profile: defaultsProfile,
want: promptkit.ExecutionTarget{
Endpoint: defaultsProfile.endpoint,
Model: defaultsProfile.model,
Temperature: 0,
MaxTokens: 0,
TopP: 1,
TopP: 0,
TimeoutSeconds: 600,
ServiceTier: defaultsProfile.serviceTier,
ReasoningEffort: defaultsProfile.reasoningEffort,
@@ -2301,6 +2301,8 @@ func TestSourceOptionsRejectInvalidInputs(t *testing.T) {
{name: "profile fs nil", opt: promptkit.WithProfileFS(nil, "profiles")},
{name: "profile fs empty root", opt: promptkit.WithProfileFS(fstest.MapFS{}, "")},
{name: "profile file empty", opt: promptkit.WithProfileFile("")},
{name: "fallback profile fs nil", opt: promptkit.WithFallbackProfileFS(nil, "profiles")},
{name: "fallback profile fs empty root", opt: promptkit.WithFallbackProfileFS(fstest.MapFS{}, "")},
{name: "schema fs nil", opt: promptkit.WithSchemaFS(nil, "schemas")},
{name: "schema fs empty root", opt: promptkit.WithSchemaFS(fstest.MapFS{}, "")},
{name: "schema file empty", opt: promptkit.WithSchemaFile("")},

View File

@@ -3,6 +3,7 @@ package promptkit
import (
"errors"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
@@ -14,6 +15,11 @@ func mapPublicError(err error) error {
if err == nil {
return nil
}
var internalCapacityError *usecase.CapacityError
if errors.As(err, &internalCapacityError) && internalCapacityError != nil &&
strings.TrimSpace(internalCapacityError.BackendID) != "" {
return &CapacityError{BackendID: internalCapacityError.BackendID}
}
publicErr := publicErrorFor(err)
if publicErr == nil {
return err

View File

@@ -20,3 +20,31 @@ func TestMapPublicErrorPreservesGenerationCancellation(t *testing.T) {
t.Fatalf("mapped error=%v, want context.Canceled", err)
}
}
func TestMapPublicErrorTranslatesCapacityError(t *testing.T) {
internalErr := &usecase.CapacityError{BackendID: "limited"}
err := mapPublicError(internalErr)
var publicErr *CapacityError
if !errors.As(err, &publicErr) || publicErr == nil {
t.Fatalf("mapped error=%v, want public CapacityError", err)
}
if publicErr.BackendID != "limited" {
t.Fatalf("mapped backend ID=%q, want limited", publicErr.BackendID)
}
if !errors.Is(err, ErrCapacityExceeded) {
t.Fatalf("mapped error=%v, want ErrCapacityExceeded", err)
}
if errors.Is(err, ErrInvalidRequest) || errors.Is(err, ErrLLMGenerate) {
t.Fatalf("mapped capacity error has an unrelated category: %v", err)
}
var leakedInternalErr *usecase.CapacityError
if errors.As(err, &leakedInternalErr) {
t.Fatalf("mapped error exposes internal CapacityError: %v", err)
}
internalErr.BackendID = "changed"
if publicErr.BackendID != "limited" {
t.Fatalf("mapped backend ID changed with source error: %q", publicErr.BackendID)
}
}

View File

@@ -124,6 +124,11 @@ func TestManagerAdmissionHonorsContextAndUnlimitedBackends(t *testing.T) {
if err != nil {
t.Fatalf("construct manager: %v", err)
}
release, err := manager.Admit(context.Background(), "limited")
if err != nil {
t.Fatalf("fill limited pool: %v", err)
}
defer release()
ctx, cancel := context.WithCancel(context.Background())
cancel()

View File

@@ -14,9 +14,6 @@ const (
ContentTypeApplicationJSON = "application/json"
OpenAIChatCompletionsPath = "/chat/completions"
ExecutionDefaultTemperature = 0.0
ExecutionDefaultMaxTokens = 0
ExecutionDefaultTopP = 1.0
ExecutionDefaultTimeoutSeconds = 600
)
@@ -26,9 +23,6 @@ var (
func ExecutionTargetDefault() domain.ExecutionTarget {
return domain.ExecutionTarget{
Temperature: ExecutionDefaultTemperature,
MaxTokens: ExecutionDefaultMaxTokens,
TopP: ExecutionDefaultTopP,
TimeoutSeconds: ExecutionDefaultTimeoutSeconds,
}
}

View File

@@ -145,6 +145,16 @@ type PromptDefinition struct {
Validation OutputContract `yaml:"validation"`
}
// PromptInspection is the resolved result of exact prompt inspection.
type PromptInspection struct {
PromptID string
PromptVersion string
PromptHash string
DefaultProfileID string
Inputs []PromptInput
OutputContract OutputContract
}
// PromptInput describes one named input expected by a prompt definition.
type PromptInput struct {
Name string `yaml:"name"`
@@ -236,6 +246,13 @@ type ExecutionTarget struct {
ExtraParams map[string]any `yaml:"extra_params" json:"extra_params"`
}
// ProfileInspection is the resolved result of exact profile inspection.
type ProfileInspection struct {
ProfileID string
EffectiveModelParams ExecutionTarget
APIKeyRequired bool
}
// OutputContract defines the requirements for the output artifact.
type OutputContract struct {
Format OutputFormat `yaml:"format"`

View File

@@ -2,7 +2,6 @@ package builtin
import (
"embed"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
)
@@ -15,17 +14,3 @@ var assets embed.FS
func NewRepository() profile.Repository {
return profile.NewFSRepository(assets, assetRoot)
}
func NewRepositoryWithPrimary(primary profile.Repository) profile.Repository {
if primary == nil {
return NewRepository()
}
return profile.NewOverlayRepository(primary, NewRepository())
}
func NewRepositoryWithDirectory(dir string) profile.Repository {
if strings.TrimSpace(dir) == "" {
return NewRepository()
}
return NewRepositoryWithPrimary(profile.NewFilesystemRepository(dir))
}

View File

@@ -2,14 +2,11 @@ package builtin
import (
"context"
"errors"
"io/fs"
"strings"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
"gopkg.in/yaml.v3"
)
@@ -91,53 +88,3 @@ func loadBuiltInProfileIDs(t *testing.T) map[string]string {
}
return ids
}
func TestRepositoryWithPrimaryUsesPrimaryBeforeBuiltIns(t *testing.T) {
repo := NewRepositoryWithPrimary(staticProfileRepo{
profiles: map[string]string{"mistral-small-3": "custom-model"},
})
p, err := repo.GetProfile(context.Background(), "mistral-small-3")
if err != nil {
t.Fatalf("expected profile to load, got %v", err)
}
if p.Model != "custom-model" {
t.Fatalf("expected primary profile to override built-in, got %+v", p)
}
}
func TestRepositoryWithPrimaryFallsBackToBuiltIns(t *testing.T) {
repo := NewRepositoryWithPrimary(staticProfileRepo{})
p, err := repo.GetProfile(context.Background(), "mistral-small-3")
if err != nil {
t.Fatalf("expected built-in profile to load, got %v", err)
}
if p.ID != "mistral-small-3" {
t.Fatalf("unexpected profile: %+v", p)
}
}
func TestRepositoryWithPrimaryDoesNotFallBackAfterPrimaryError(t *testing.T) {
repo := NewRepositoryWithPrimary(staticProfileRepo{err: profile.ErrInvalidProfile})
_, err := repo.GetProfile(context.Background(), "mistral-small-3")
if !errors.Is(err, profile.ErrInvalidProfile) {
t.Fatalf("expected primary error, got %v", err)
}
}
type staticProfileRepo struct {
profiles map[string]string
err error
}
func (r staticProfileRepo) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) {
if r.err != nil {
return nil, r.err
}
if model, ok := r.profiles[id]; ok {
return &domain.ExecutionProfile{ID: id, Endpoint: "http://primary/v1", Model: model}, nil
}
return nil, profile.ErrProfileNotFound
}

View File

@@ -0,0 +1,24 @@
package usecase
import (
"fmt"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
)
// CapacityError identifies bounded admission rejected for one selected backend.
type CapacityError struct {
BackendID string
}
func (e *CapacityError) Error() string {
if e == nil || strings.TrimSpace(e.BackendID) == "" {
return capacity.ErrCapacityExceeded.Error()
}
return fmt.Sprintf("backend %q admission: %v", e.BackendID, capacity.ErrCapacityExceeded)
}
func (e *CapacityError) Unwrap() error {
return capacity.ErrCapacityExceeded
}

View File

@@ -0,0 +1,103 @@
package usecase
import (
"context"
"errors"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
type resolvedProfileSelection struct {
id string
profile *domain.ExecutionProfile
backend *domain.Backend
}
func (r *Runner) resolveProfileSelection(
ctx context.Context,
profileID string,
) (*resolvedProfileSelection, error) {
normalizedID := strings.TrimSpace(profileID)
if normalizedID == "" {
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
}
if r == nil || r.profiles == nil {
return nil, fmt.Errorf("%w: profile repository is not configured", ErrProfileLoad)
}
selectedProfile, err := r.profiles.GetProfile(ctx, normalizedID)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
}
if selectedProfile == nil {
return nil, fmt.Errorf("%w: profile repository returned nil profile", ErrProfileLoad)
}
profileValue := *selectedProfile
profileValue.BackendID = strings.TrimSpace(profileValue.BackendID)
var selectedBackend *domain.Backend
if profileValue.BackendID != "" {
if r.backends == nil {
return nil, fmt.Errorf("%w: backend %q cannot be resolved", ErrProfileLoad, profileValue.BackendID)
}
backendValue, err := r.backends.GetBackend(profileValue.BackendID)
if err != nil {
return nil, fmt.Errorf("%w: backend %q: %w", ErrProfileLoad, profileValue.BackendID, err)
}
selectedBackend = &backendValue
}
return &resolvedProfileSelection{
id: normalizedID,
profile: &profileValue,
backend: selectedBackend,
}, nil
}
func validateResolvedExecutionTarget(target domain.ExecutionTarget) error {
if strings.TrimSpace(target.Endpoint) == "" {
return errors.New("execution endpoint is required")
}
if strings.TrimSpace(target.Model) == "" {
return errors.New("execution model is required")
}
return nil
}
// InspectProfile resolves one explicit profile without prompt or execution work.
func (r *Runner) InspectProfile(
ctx context.Context,
profileID string,
) (*domain.ProfileInspection, error) {
normalizedID := strings.TrimSpace(profileID)
if normalizedID == "" {
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
}
select {
case <-ctx.Done():
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, ctx.Err())
default:
}
selection, err := r.resolveProfileSelection(ctx, normalizedID)
if err != nil {
return nil, err
}
target, _, err := resolveExecutionTarget(selection.backend, selection.profile, nil)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
}
if err := validateResolvedExecutionTarget(target); err != nil {
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
}
target.APIKey = ""
return &domain.ProfileInspection{
ProfileID: selection.id,
EffectiveModelParams: target,
APIKeyRequired: target.APIKeyRequired,
}, nil
}

View File

@@ -0,0 +1,213 @@
package usecase
import (
"context"
"errors"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
)
type inspectionProfileRepository struct {
profile *domain.ExecutionProfile
err error
calls int
id string
}
func (r *inspectionProfileRepository) GetProfile(
_ context.Context,
id string,
) (*domain.ExecutionProfile, error) {
r.calls++
r.id = id
if r.err != nil {
return nil, r.err
}
return r.profile, nil
}
type inspectionBackendResolver struct {
backend domain.Backend
err error
calls int
id string
}
func (r *inspectionBackendResolver) GetBackend(id string) (domain.Backend, error) {
r.calls++
r.id = id
if r.err != nil {
return domain.Backend{}, r.err
}
return r.backend, nil
}
func TestRunnerInspectProfileResolvesProfileAndBackendOnce(t *testing.T) {
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
ID: "profile",
BackendID: " backend ",
Model: "profile-model",
Temperature: 0.4,
MaxTokens: 32,
TimeoutSeconds: 45,
ServiceTier: "priority",
ReasoningEffort: "high",
ExtraParams: map[string]any{
"profile": "value",
},
}}
backends := &inspectionBackendResolver{backend: domain.Backend{
ID: "backend",
Endpoint: "https://backend.example/v1",
APIKeyEnv: "BACKEND_KEY",
ExtraParams: map[string]any{
"backend": "value",
},
}}
runner := &Runner{profiles: profiles, backends: backends}
inspection, err := runner.InspectProfile(context.Background(), " profile ")
if err != nil {
t.Fatalf("inspect profile: %v", err)
}
if profiles.calls != 1 || profiles.id != "profile" {
t.Fatalf("profile lookup=(calls=%d id=%q), want one exact lookup", profiles.calls, profiles.id)
}
if backends.calls != 1 || backends.id != "backend" {
t.Fatalf("backend lookup=(calls=%d id=%q), want one exact lookup", backends.calls, backends.id)
}
if profiles.profile.BackendID != " backend " {
t.Fatalf("inspection mutated repository profile backend: %q", profiles.profile.BackendID)
}
wantTarget := domain.ExecutionTarget{
BackendID: "backend",
Endpoint: "https://backend.example/v1",
Model: "profile-model",
Temperature: 0.4,
MaxTokens: 32,
TopP: defaults.ExecutionTargetDefault().TopP,
TimeoutSeconds: 45,
ServiceTier: "priority",
ReasoningEffort: "high",
APIKeyEnv: "BACKEND_KEY",
ExtraParams: map[string]any{
"profile": "value",
},
}
if inspection.ProfileID != "profile" || inspection.APIKeyRequired ||
!reflect.DeepEqual(inspection.EffectiveModelParams, wantTarget) {
t.Fatalf("inspection=%#v, want profile=%q target=%#v", inspection, "profile", wantTarget)
}
}
func TestRunnerInspectProfileDoesNotNeedExecutionCollaboratorsOrCredentials(t *testing.T) {
t.Setenv("PROMPTKIT_INSPECTION_TEST_KEY", "")
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
ID: "endpoint-only",
Endpoint: "https://profile.example/v1",
Model: "profile-model",
APIKeyEnv: "PROMPTKIT_INSPECTION_TEST_KEY",
}}
runner := &Runner{profiles: profiles}
inspection, err := runner.InspectProfile(context.Background(), "endpoint-only")
if err != nil {
t.Fatalf("inspect endpoint-only profile: %v", err)
}
if inspection.EffectiveModelParams.BackendID != "" ||
inspection.EffectiveModelParams.APIKeyEnv != "PROMPTKIT_INSPECTION_TEST_KEY" ||
inspection.APIKeyRequired {
t.Fatalf("unexpected endpoint-only inspection: %#v", inspection)
}
}
func TestRunnerInspectProfileDirectCredentialRequirementClearsBackendEnvironment(t *testing.T) {
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
ID: "direct-key",
BackendID: "backend",
Model: "profile-model",
APIKeyRequired: true,
}}
backends := &inspectionBackendResolver{backend: domain.Backend{
ID: "backend",
Endpoint: "https://backend.example/v1",
APIKeyEnv: "BACKEND_KEY",
}}
inspection, err := (&Runner{profiles: profiles, backends: backends}).InspectProfile(
context.Background(),
"direct-key",
)
if err != nil {
t.Fatalf("inspect direct-key profile: %v", err)
}
if !inspection.APIKeyRequired || inspection.EffectiveModelParams.APIKeyEnv != "" {
t.Fatalf("credential requirement was not resolved exclusively: %#v", inspection)
}
}
func TestRunnerInspectProfileClassifiesFailuresWithoutRepositoryWorkAfterCancellation(t *testing.T) {
t.Run("blank ID", func(t *testing.T) {
profiles := &inspectionProfileRepository{}
_, err := (&Runner{profiles: profiles}).InspectProfile(context.Background(), " \t ")
if !errors.Is(err, ErrInvalidRequest) || profiles.calls != 0 {
t.Fatalf("blank inspection=(%v, calls=%d), want invalid request without lookup", err, profiles.calls)
}
})
t.Run("canceled context", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
profiles := &inspectionProfileRepository{}
_, err := (&Runner{profiles: profiles}).InspectProfile(ctx, "profile")
if !errors.Is(err, ErrProfileLoad) || !errors.Is(err, context.Canceled) || profiles.calls != 0 {
t.Fatalf("canceled inspection=(%v, calls=%d), want profile load and context identities without lookup", err, profiles.calls)
}
})
t.Run("missing profile", func(t *testing.T) {
profiles := &inspectionProfileRepository{err: profile.ErrProfileNotFound}
_, err := (&Runner{profiles: profiles}).InspectProfile(context.Background(), "missing")
if !errors.Is(err, ErrProfileLoad) || !errors.Is(err, profile.ErrProfileNotFound) {
t.Fatalf("missing profile error=%v, want profile load and not-found identities", err)
}
})
t.Run("unknown backend", func(t *testing.T) {
backendErr := errors.New("unknown backend")
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
ID: "profile", BackendID: "backend", Model: "profile-model",
}}
backends := &inspectionBackendResolver{err: backendErr}
_, err := (&Runner{profiles: profiles, backends: backends}).InspectProfile(context.Background(), "profile")
if !errors.Is(err, ErrProfileLoad) || !errors.Is(err, backendErr) {
t.Fatalf("unknown backend error=%v, want profile load and backend identities", err)
}
})
t.Run("defensive invalid dependencies", func(t *testing.T) {
cases := []struct {
name string
runner *Runner
}{
{name: "nil repository", runner: &Runner{}},
{name: "nil profile", runner: &Runner{profiles: &inspectionProfileRepository{}}},
{name: "invalid target", runner: &Runner{profiles: &inspectionProfileRepository{
profile: &domain.ExecutionProfile{ID: "profile", Endpoint: "https://profile.example/v1"},
}}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
_, err := tc.runner.InspectProfile(context.Background(), "profile")
if !errors.Is(err, ErrProfileLoad) {
t.Fatalf("inspection error=%v, want profile load", err)
}
})
}
})
}

View File

@@ -0,0 +1,73 @@
package usecase
import (
"context"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
type resolvedPromptDefinition struct {
definition *domain.PromptDefinition
hash string
}
func (r *Runner) resolvePromptDefinition(
ctx context.Context,
promptID string,
promptVersion string,
) (*resolvedPromptDefinition, error) {
if strings.TrimSpace(promptID) == "" {
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
}
if r == nil || r.promptDefs == nil {
return nil, fmt.Errorf("%w: prompt repository is not configured", ErrPromptLoad)
}
definition, err := r.promptDefs.GetPromptDefinition(ctx, promptID, promptVersion)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
}
if definition == nil {
return nil, fmt.Errorf("%w: prompt repository returned nil definition", ErrPromptLoad)
}
hash, err := hashPromptDefinition(definition)
if err != nil {
return nil, fmt.Errorf("%w: failed to hash prompt definition: %v", ErrPromptLoad, err)
}
return &resolvedPromptDefinition{definition: definition, hash: hash}, nil
}
// InspectPrompt resolves one explicit prompt without execution work.
func (r *Runner) InspectPrompt(
ctx context.Context,
promptID string,
promptVersion string,
) (*domain.PromptInspection, error) {
if strings.TrimSpace(promptID) == "" {
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
}
select {
case <-ctx.Done():
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, ctx.Err())
default:
}
selection, err := r.resolvePromptDefinition(ctx, promptID, promptVersion)
if err != nil {
return nil, err
}
inputs := make([]domain.PromptInput, len(selection.definition.Inputs))
copy(inputs, selection.definition.Inputs)
return &domain.PromptInspection{
PromptID: selection.definition.ID,
PromptVersion: selection.definition.Version,
PromptHash: selection.hash,
DefaultProfileID: selection.definition.DefaultProfile,
Inputs: inputs,
OutputContract: selection.definition.Validation,
}, nil
}

View File

@@ -0,0 +1,157 @@
package usecase
import (
"context"
"errors"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
)
type inspectionPromptRepository struct {
definition *domain.PromptDefinition
err error
calls int
id string
version string
}
func (r *inspectionPromptRepository) GetPromptDefinition(
_ context.Context,
id string,
version string,
) (*domain.PromptDefinition, error) {
r.calls++
r.id = id
r.version = version
if r.err != nil {
return nil, r.err
}
return r.definition, nil
}
func TestRunnerInspectPromptResolvesOneDefinitionWithoutExecutionCollaborators(t *testing.T) {
definition := &domain.PromptDefinition{
ID: "normalized.prompt",
Version: "1.2.3",
DefaultProfile: "not-resolved",
Inputs: []domain.PromptInput{
{Name: "document", Required: true, ContentType: "text/plain", Description: "Source document."},
{Name: "audience", ContentType: "text/plain", Description: "Intended reader."},
},
Validation: domain.OutputContract{
Format: domain.FormatJSON,
ValidationMode: domain.ValidationJSONSchema,
SchemaPath: "schemas/result.json",
RepairAttempts: 2,
},
}
repository := &inspectionPromptRepository{definition: definition}
runner := &Runner{promptDefs: repository}
inspection, err := runner.InspectPrompt(context.Background(), " prompt-id ", " version ")
if err != nil {
t.Fatalf("inspect prompt: %v", err)
}
wantHash, err := hashPromptDefinition(definition)
if err != nil {
t.Fatalf("hash prompt definition: %v", err)
}
if repository.calls != 1 || repository.id != " prompt-id " || repository.version != " version " {
t.Fatalf("prompt lookup=(calls=%d id=%q version=%q), want one unchanged lookup", repository.calls, repository.id, repository.version)
}
if inspection.PromptID != definition.ID ||
inspection.PromptVersion != definition.Version ||
inspection.PromptHash != wantHash ||
inspection.DefaultProfileID != definition.DefaultProfile ||
!reflect.DeepEqual(inspection.Inputs, definition.Inputs) ||
inspection.OutputContract != definition.Validation {
t.Fatalf("inspection=%#v, want definition metadata", inspection)
}
inspection.Inputs[0].Name = "changed"
second, err := runner.InspectPrompt(context.Background(), " prompt-id ", " version ")
if err != nil {
t.Fatalf("inspect prompt again: %v", err)
}
if definition.Inputs[0].Name != "document" || second.Inputs[0].Name != "document" {
t.Fatalf("inspection input mutation escaped caller result: definition=%#v next=%#v", definition.Inputs, second.Inputs)
}
}
func TestRunnerInspectPromptClassifiesFailuresWithoutRepositoryWorkAfterCancellation(t *testing.T) {
t.Run("blank ID", func(t *testing.T) {
repository := &inspectionPromptRepository{}
_, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), " \t ", "version")
if !errors.Is(err, ErrInvalidRequest) || repository.calls != 0 {
t.Fatalf("blank inspection=(%v, calls=%d), want invalid request without lookup", err, repository.calls)
}
})
t.Run("canceled context", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
repository := &inspectionPromptRepository{}
_, err := (&Runner{promptDefs: repository}).InspectPrompt(ctx, "prompt", "version")
if !errors.Is(err, ErrPromptLoad) || !errors.Is(err, context.Canceled) || repository.calls != 0 {
t.Fatalf("canceled inspection=(%v, calls=%d), want prompt load and context identities without lookup", err, repository.calls)
}
})
t.Run("missing prompt", func(t *testing.T) {
repository := &inspectionPromptRepository{err: promptdef.ErrPromptDefinitionNotFound}
_, err := (&Runner{promptDefs: repository}).InspectPrompt(context.Background(), "missing", "version")
if !errors.Is(err, ErrPromptLoad) || !errors.Is(err, promptdef.ErrPromptDefinitionNotFound) {
t.Fatalf("missing prompt error=%v, want prompt load and not-found identities", err)
}
})
t.Run("defensive prompt dependencies", func(t *testing.T) {
var nilRunner *Runner
cases := []struct {
name string
runner *Runner
}{
{name: "nil runner", runner: nilRunner},
{name: "nil repository", runner: &Runner{}},
{name: "nil definition", runner: &Runner{promptDefs: &inspectionPromptRepository{}}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
_, err := tc.runner.InspectPrompt(context.Background(), "prompt", "version")
if !errors.Is(err, ErrPromptLoad) {
t.Fatalf("inspection error=%v, want prompt load", err)
}
})
}
})
}
func TestRunnerPrepareUsesThePromptInspectionSelectionAndHash(t *testing.T) {
definition := promptDef(domain.FormatMarkdown, domain.ValidationBasic, 0)
repository := &fakePromptRepo{def: definition}
runner := NewRunner(
repository,
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
&fakeArtifactReader{},
&fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hello"}}}},
nil,
nil,
nil,
)
inspection, err := runner.InspectPrompt(context.Background(), definition.ID, definition.Version)
if err != nil {
t.Fatalf("inspect prompt: %v", err)
}
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: definition.ID, PromptVersion: definition.Version, ProfileID: "exec"})
if err != nil {
t.Fatalf("prepare prompt: %v", err)
}
if inspection.PromptHash != prepared.PromptHash {
t.Fatalf("inspection hash=%q, preparation hash=%q", inspection.PromptHash, prepared.PromptHash)
}
}

View File

@@ -266,14 +266,12 @@ func (r *Runner) resolvePreparation(
return nil, fmt.Errorf("%w: session_id: %v", ErrInvalidRequest, err)
}
def, err := r.promptDefs.GetPromptDefinition(ctx, req.PromptID, req.PromptVersion)
promptSelection, err := r.resolvePromptDefinition(ctx, req.PromptID, req.PromptVersion)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
}
promptDefinitionHash, err := hashPromptDefinition(def)
if err != nil {
return nil, fmt.Errorf("%w: failed to hash prompt definition: %v", ErrPromptLoad, err)
return nil, err
}
def := promptSelection.definition
promptDefinitionHash := promptSelection.hash
selectedProfileID := strings.TrimSpace(req.ProfileID)
if selectedProfileID == "" {
@@ -283,34 +281,18 @@ func (r *Runner) resolvePreparation(
return nil, fmt.Errorf("%w: %w: profile id is required either in request or prompt default_profile", ErrInvalidRequest, ErrProfileRequired)
}
execProfile, err := r.profiles.GetProfile(ctx, selectedProfileID)
selection, err := r.resolveProfileSelection(ctx, selectedProfileID)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
return nil, err
}
var selectedBackend *domain.Backend
if backendID := strings.TrimSpace(execProfile.BackendID); backendID != "" {
execProfile.BackendID = backendID
if r.backends == nil {
return nil, fmt.Errorf("%w: backend %q cannot be resolved", ErrProfileLoad, backendID)
}
resolvedBackend, resolveErr := r.backends.GetBackend(backendID)
if resolveErr != nil {
return nil, fmt.Errorf("%w: backend %q: %w", ErrProfileLoad, backendID, resolveErr)
}
selectedBackend = &resolvedBackend
}
effectiveModel, targetPresence, err := resolveExecutionTarget(selectedBackend, execProfile, req.Execution)
effectiveModel, targetPresence, err := resolveExecutionTarget(selection.backend, selection.profile, req.Execution)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
}
effectiveModel.APIKey = req.APIKey
if strings.TrimSpace(effectiveModel.Endpoint) == "" {
return nil, fmt.Errorf("%w: execution endpoint is required", ErrInvalidRequest)
}
if strings.TrimSpace(effectiveModel.Model) == "" {
return nil, fmt.Errorf("%w: execution model is required", ErrInvalidRequest)
if err := validateResolvedExecutionTarget(effectiveModel); err != nil {
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
}
if err := validateAPIKey(effectiveModel.APIKeyEnv, effectiveModel.APIKey, effectiveModel.APIKeyRequired); err != nil {
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
@@ -321,7 +303,7 @@ func (r *Runner) resolvePreparation(
definition: def,
directSessionID: directSessionID,
promptDefinitionHash: promptDefinitionHash,
selectedProfileID: selectedProfileID,
selectedProfileID: selection.id,
effectiveModel: effectiveModel,
targetPresence: targetPresence,
effectiveContract: effectiveContract,
@@ -408,8 +390,8 @@ func (r *Runner) admitRun(ctx context.Context, backendID string) (func(), error)
}
release, err := r.admitter.Admit(ctx, backendID)
if err != nil {
if errors.Is(err, capacity.ErrCapacityExceeded) {
return nil, fmt.Errorf("backend %q admission: %w", backendID, err)
if errors.Is(err, capacity.ErrCapacityExceeded) && strings.TrimSpace(backendID) != "" {
return nil, &CapacityError{BackendID: backendID}
}
return nil, err
}

View File

@@ -1357,14 +1357,14 @@ func TestRunnerAdmissionUsesResolvedBackendIdentity(t *testing.T) {
func TestRunnerAdmissionFailureSkipsCompletionCollaborators(t *testing.T) {
tests := []struct {
name string
admissionError error
wantBackendContext bool
name string
admissionError error
wantCapacityType bool
}{
{
name: "capacity exhausted",
admissionError: capacity.ErrCapacityExceeded,
wantBackendContext: true,
name: "capacity exhausted",
admissionError: capacity.ErrCapacityExceeded,
wantCapacityType: true,
},
{
name: "context canceled",
@@ -1414,8 +1414,16 @@ func TestRunnerAdmissionFailureSkipsCompletionCollaborators(t *testing.T) {
if errors.Is(err, ErrInvalidRequest) || errors.Is(err, ErrLLMGenerate) {
t.Fatalf("admission error was recategorized: %v", err)
}
if tc.wantBackendContext && !strings.Contains(err.Error(), "custom") {
t.Fatalf("capacity error lacks backend context: %v", err)
var capacityErr *CapacityError
if tc.wantCapacityType {
if !errors.As(err, &capacityErr) {
t.Fatalf("capacity error=%v, want internal typed identity", err)
}
if capacityErr.BackendID != "custom" {
t.Fatalf("capacity backend ID=%q, want custom", capacityErr.BackendID)
}
} else if errors.As(err, &capacityErr) {
t.Fatalf("non-capacity admission error exposed typed capacity identity: %v", err)
}
if !reflect.DeepEqual(admitter.backendIDs, []string{"custom"}) {
t.Fatalf("admitted backend IDs=%#v, want custom", admitter.backendIDs)
@@ -1682,36 +1690,6 @@ func TestRunnerRunSelectedProfileBeatsBuiltInDefault(t *testing.T) {
}
}
func TestRunnerRunBuiltInDefaultsUsedWhenProfileOmitsOptionalFields(t *testing.T) {
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"},
}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if res.EffectiveModelParams.Temperature != defaults.ExecutionDefaultTemperature {
t.Fatalf("expected default temperature %v, got %v", defaults.ExecutionDefaultTemperature, res.EffectiveModelParams.Temperature)
}
if res.EffectiveModelParams.TopP != defaults.ExecutionDefaultTopP {
t.Fatalf("expected default top_p %v, got %v", defaults.ExecutionDefaultTopP, res.EffectiveModelParams.TopP)
}
if res.EffectiveModelParams.MaxTokens != defaults.ExecutionDefaultMaxTokens {
t.Fatalf("expected default max_tokens %d, got %d", defaults.ExecutionDefaultMaxTokens, res.EffectiveModelParams.MaxTokens)
}
if res.EffectiveModelParams.TimeoutSeconds != defaults.ExecutionDefaultTimeoutSeconds {
t.Fatalf("expected default timeout_seconds %d, got %d", defaults.ExecutionDefaultTimeoutSeconds, res.EffectiveModelParams.TimeoutSeconds)
}
}
func TestRunnerRunAPIKeyEnvResolvesFromEnvironment(t *testing.T) {
t.Setenv("PROMPTKIT_TEST_API_KEY", "secret")
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}

View File

@@ -560,6 +560,11 @@ func TestPreparedExecutionCredentialCapacityAndTimingBoundaries(t *testing.T) {
if result, err := engine.RunPrepared(context.Background(), prepared); result != nil ||
!errors.Is(err, promptkit.ErrCapacityExceeded) {
t.Fatalf("capacity execution=(%+v, %v), want ErrCapacityExceeded", result, err)
} else {
var capacityErr *promptkit.CapacityError
if !errors.As(err, &capacityErr) || capacityErr == nil || capacityErr.BackendID != "limited" {
t.Fatalf("capacity execution=%v, want limited CapacityError", err)
}
}
if result, err := engine.RunPrepared(context.Background(), prepared); result != nil ||
!errors.Is(err, promptkit.ErrInvalidRequest) {

View File

@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"fmt"
"io/fs"
"reflect"
"strings"
"sync"
@@ -16,6 +17,380 @@ import (
"gitea.maximumdirect.net/eric/promptkit"
)
type inspectionCountingFS struct {
opens atomic.Int64
}
func (f *inspectionCountingFS) Open(string) (fs.File, error) {
f.opens.Add(1)
return nil, fs.ErrNotExist
}
func TestInspectProfileResolvesCredentialStatesWithoutPromptOrGeneration(t *testing.T) {
const environmentName = "PROMPTKIT_INSPECTION_ABSENT_KEY"
t.Setenv(environmentName, "")
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}}
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(fstest.MapFS{}, "."),
promptkit.WithProfileFS(fstest.MapFS{
"environment.yaml": &fstest.MapFile{Data: []byte(`id: environment
endpoint: http://environment.example/v1
model: environment-model
api_key_env: PROMPTKIT_INSPECTION_ABSENT_KEY
`)},
}, "."),
promptkit.WithProfiles(
promptkit.Profile{ID: "direct", Endpoint: "http://direct.example/v1", Model: "direct-model", APIKeyRequired: true},
promptkit.Profile{ID: "none", Endpoint: "http://none.example/v1", Model: "none-model"},
),
promptkit.WithLLMClient(client),
)
if err != nil {
t.Fatalf("construct inspection engine: %v", err)
}
for _, tc := range []struct {
profileID string
wantEnv string
wantDirectKey bool
wantEndpoint string
}{
{profileID: " environment ", wantEnv: environmentName, wantEndpoint: "http://environment.example/v1"},
{profileID: "direct", wantDirectKey: true, wantEndpoint: "http://direct.example/v1"},
{profileID: "none", wantEndpoint: "http://none.example/v1"},
} {
t.Run(tc.profileID, func(t *testing.T) {
inspection, err := engine.InspectProfile(context.Background(), tc.profileID)
if err != nil {
t.Fatalf("inspect profile: %v", err)
}
if inspection.ProfileID != strings.TrimSpace(tc.profileID) ||
inspection.EffectiveModelParams.Endpoint != tc.wantEndpoint ||
inspection.EffectiveModelParams.BackendID != "" ||
inspection.EffectiveModelParams.APIKeyEnv != tc.wantEnv ||
inspection.APIKeyRequired != tc.wantDirectKey {
t.Fatalf("unexpected inspection: %#v", inspection)
}
})
}
if len(client.requests) != 0 {
t.Fatalf("inspection invoked the model client %d times", len(client.requests))
}
}
func TestInspectProfilePreservesPublicErrorIdentities(t *testing.T) {
newEngine := func(t *testing.T, options ...promptkit.Option) *promptkit.Engine {
t.Helper()
engine, err := promptkit.NewEngine(
promptkit.Config{},
append([]promptkit.Option{promptkit.WithPromptFS(fstest.MapFS{}, ".")}, options...)...,
)
if err != nil {
t.Fatalf("construct inspection engine: %v", err)
}
return engine
}
var nilEngine *promptkit.Engine
if result, err := nilEngine.InspectProfile(context.Background(), "profile"); result != nil ||
!errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("nil engine result=(%#v, %v), want ErrInvalidConfig", result, err)
}
valid := newEngine(t, promptkit.WithProfiles(promptkit.Profile{
ID: "profile", Endpoint: "http://profile.example/v1", Model: "model",
}))
if result, err := valid.InspectProfile(context.Background(), " \t "); result != nil ||
!errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("blank profile result=(%#v, %v), want ErrInvalidRequest", result, err)
}
if result, err := valid.InspectProfile(context.Background(), "missing"); result != nil ||
!errors.Is(err, promptkit.ErrProfileNotFound) || errors.Is(err, promptkit.ErrProfileLoad) {
t.Fatalf("missing profile result=(%#v, %v), want only ErrProfileNotFound", result, err)
}
malformed := newEngine(t, promptkit.WithProfileFS(fstest.MapFS{
"broken.yaml": &fstest.MapFile{Data: []byte("id: broken\nendpoint: http://broken.example/v1\nmodel: model\nunknown: value\n")},
}, "."))
if result, err := malformed.InspectProfile(context.Background(), "broken"); result != nil ||
!errors.Is(err, promptkit.ErrProfileLoad) {
t.Fatalf("malformed profile result=(%#v, %v), want ErrProfileLoad", result, err)
}
unknownBackend := newEngine(t, promptkit.WithProfiles(promptkit.Profile{
ID: "unknown-backend", BackendID: "unknown", Model: "model",
}))
if result, err := unknownBackend.InspectProfile(context.Background(), "unknown-backend"); result != nil ||
!errors.Is(err, promptkit.ErrProfileLoad) {
t.Fatalf("unknown backend result=(%#v, %v), want ErrProfileLoad", result, err)
}
countingFS := &inspectionCountingFS{}
canceled := newEngine(t, promptkit.WithProfileFS(countingFS, "."))
ctx, cancel := context.WithCancel(context.Background())
cancel()
if result, err := canceled.InspectProfile(ctx, "profile"); result != nil ||
!errors.Is(err, promptkit.ErrProfileLoad) || !errors.Is(err, context.Canceled) || countingFS.opens.Load() != 0 {
t.Fatalf("canceled inspection result=(%#v, %v), opens=%d", result, err, countingFS.opens.Load())
}
}
func TestInspectProfileReturnsIndependentTargetMatchingPreparation(t *testing.T) {
extraParams := map[string]any{
"nested": map[string]any{"value": "original"},
}
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
promptkit.WithProfiles(promptkit.Profile{
ID: "profile", Endpoint: "http://profile.example/v1", Model: "model", ExtraParams: extraParams,
}),
)
if err != nil {
t.Fatalf("construct inspection engine: %v", err)
}
first, err := engine.InspectProfile(context.Background(), "profile")
if err != nil {
t.Fatalf("first inspection: %v", err)
}
first.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["value"] = "changed"
first.EffectiveModelParams.ExtraParams["later"] = true
second, err := engine.InspectProfile(context.Background(), "profile")
if err != nil {
t.Fatalf("second inspection: %v", err)
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatalf("prepare after inspection mutation: %v", err)
}
for _, target := range []promptkit.ExecutionTarget{second.EffectiveModelParams, prepared.EffectiveModelParams} {
if target.ExtraParams["nested"].(map[string]any)["value"] != "original" || target.ExtraParams["later"] != nil {
t.Fatalf("inspection mutation reached engine-owned target: %#v", target.ExtraParams)
}
}
if !reflect.DeepEqual(second.EffectiveModelParams, prepared.EffectiveModelParams) {
t.Fatalf("inspection target=%#v, preparation target=%#v", second.EffectiveModelParams, prepared.EffectiveModelParams)
}
}
func TestInspectPromptReturnsDeclaredMetadataWithoutExecutionWork(t *testing.T) {
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}}
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(fstest.MapFS{
"report-v1.yaml": &fstest.MapFile{Data: []byte(`id: report
version: "1.0.0"
messages:
- role: user
content: old report
output:
format: text
validation_mode: none
`)},
"report-v2.yaml": &fstest.MapFile{Data: []byte(`id: report
version: "2.0.0"
default_profile: missing-profile
inputs:
- name: location
required: true
content_type: text/plain
description: Forecast location.
- name: units
content_type: text/plain
description: Unit preference.
messages:
- role: user
content_file: messages/report.md
output:
format: json
validation_mode: json_schema
schema_path: schemas/report.json
`)},
"messages/report.md": &fstest.MapFile{Data: []byte("rendered report body is not returned")},
}, "."),
promptkit.WithLLMClient(client),
)
if err != nil {
t.Fatalf("construct prompt-inspection engine: %v", err)
}
inspection, err := engine.InspectPrompt(context.Background(), "report", "2.0.0")
if err != nil {
t.Fatalf("inspect prompt: %v", err)
}
if inspection.PromptID != "report" ||
inspection.PromptVersion != "2.0.0" ||
inspection.PromptHash == "" ||
inspection.DefaultProfileID != "missing-profile" ||
inspection.OutputContract != (promptkit.OutputContract{
Format: promptkit.FormatJSON,
ValidationMode: promptkit.ValidationJSONSchema,
SchemaPath: "schemas/report.json",
}) {
t.Fatalf("unexpected inspection metadata: %#v", inspection)
}
wantInputs := []promptkit.PromptInputDefinition{
{Name: "location", Required: true, ContentType: "text/plain", Description: "Forecast location."},
{Name: "units", ContentType: "text/plain", Description: "Unit preference."},
}
if !reflect.DeepEqual(inspection.Inputs, wantInputs) {
t.Fatalf("inspection inputs=%#v, want %#v", inspection.Inputs, wantInputs)
}
if len(client.requests) != 0 {
t.Fatalf("inspection invoked the model client %d times", len(client.requests))
}
}
func TestInspectPromptPreservesPublicErrorIdentities(t *testing.T) {
newEngine := func(t *testing.T, source fstest.MapFS) *promptkit.Engine {
t.Helper()
engine, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithPromptFS(source, "."))
if err != nil {
t.Fatalf("construct prompt-inspection engine: %v", err)
}
return engine
}
validSource := fstest.MapFS{
"prompt.yaml": &fstest.MapFile{Data: []byte(`id: prompt
version: "1"
messages:
- role: user
content: body
output:
format: text
validation_mode: none
`)},
}
var nilEngine *promptkit.Engine
if result, err := nilEngine.InspectPrompt(context.Background(), "prompt", "1"); result != nil ||
!errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("nil engine result=(%#v, %v), want ErrInvalidConfig", result, err)
}
valid := newEngine(t, validSource)
if result, err := valid.InspectPrompt(context.Background(), " \t ", "1"); result != nil ||
!errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("blank prompt result=(%#v, %v), want ErrInvalidRequest", result, err)
}
if result, err := valid.InspectPrompt(context.Background(), "missing", "1"); result != nil ||
!errors.Is(err, promptkit.ErrPromptNotFound) || errors.Is(err, promptkit.ErrPromptLoad) {
t.Fatalf("missing prompt result=(%#v, %v), want only ErrPromptNotFound", result, err)
}
if result, err := valid.InspectPrompt(context.Background(), "prompt", "missing"); result != nil ||
!errors.Is(err, promptkit.ErrPromptNotFound) || errors.Is(err, promptkit.ErrPromptLoad) {
t.Fatalf("missing version result=(%#v, %v), want only ErrPromptNotFound", result, err)
}
ambiguous := newEngine(t, fstest.MapFS{
"one.yaml": &fstest.MapFile{Data: []byte(`id: prompt
version: "1"
messages:
- role: user
content: first
output:
format: text
validation_mode: none
`)},
"two.yaml": &fstest.MapFile{Data: []byte(`id: prompt
version: "2"
messages:
- role: user
content: second
output:
format: text
validation_mode: none
`)},
})
if result, err := ambiguous.InspectPrompt(context.Background(), "prompt", ""); result != nil ||
!errors.Is(err, promptkit.ErrPromptLoad) {
t.Fatalf("ambiguous prompt result=(%#v, %v), want ErrPromptLoad", result, err)
}
for name, source := range map[string]fstest.MapFS{
"malformed definition": {
"broken.yaml": &fstest.MapFile{Data: []byte("id: broken\nversion: \"1\"\nunknown: value\n")},
},
"missing content file": {
"broken.yaml": &fstest.MapFile{Data: []byte(`id: broken
version: "1"
messages:
- role: user
content_file: missing.md
output:
format: text
validation_mode: none
`)},
},
} {
t.Run(name, func(t *testing.T) {
if result, err := newEngine(t, source).InspectPrompt(context.Background(), "broken", "1"); result != nil ||
!errors.Is(err, promptkit.ErrPromptLoad) {
t.Fatalf("broken prompt result=(%#v, %v), want ErrPromptLoad", result, err)
}
})
}
countingFS := &inspectionCountingFS{}
canceled, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithPromptFS(countingFS, "."))
if err != nil {
t.Fatalf("construct canceled prompt-inspection engine: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if result, err := canceled.InspectPrompt(ctx, "prompt", "1"); result != nil ||
!errors.Is(err, promptkit.ErrPromptLoad) || !errors.Is(err, context.Canceled) || countingFS.opens.Load() != 0 {
t.Fatalf("canceled inspection result=(%#v, %v), opens=%d", result, err, countingFS.opens.Load())
}
}
func TestInspectPromptReturnsIndependentMetadataMatchingPreparation(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(fstest.MapFS{
"prompt.yaml": &fstest.MapFile{Data: []byte(`id: prompt
version: "1"
default_profile: profile
inputs:
- name: subject
content_type: text/plain
description: Summary subject.
messages:
- role: user
content: summarize
output:
format: markdown
validation_mode: basic
`)},
}, "."),
promptkit.WithProfiles(promptkit.Profile{
ID: "profile", Endpoint: "http://profile.example/v1", Model: "model",
}),
)
if err != nil {
t.Fatalf("construct prompt-inspection engine: %v", err)
}
first, err := engine.InspectPrompt(context.Background(), "prompt", "1")
if err != nil {
t.Fatalf("first inspection: %v", err)
}
first.Inputs[0].Name = "changed"
first.OutputContract.SchemaPath = "changed.json"
second, err := engine.InspectPrompt(context.Background(), "prompt", "1")
if err != nil {
t.Fatalf("second inspection: %v", err)
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt", PromptVersion: "1"})
if err != nil {
t.Fatalf("prepare after inspection mutation: %v", err)
}
if second.Inputs[0].Name != "subject" || second.OutputContract.SchemaPath != "" ||
prepared.OutputContract.SchemaPath != "" || second.PromptHash != prepared.PromptHash {
t.Fatalf("inspection mutation reached engine-owned prompt metadata: inspection=%#v prepared=%#v", second, prepared)
}
}
func TestPreparedRunJSONOmitsZeroTimingValues(t *testing.T) {
payload, err := json.Marshal(promptkit.PreparedRun{})
if err != nil {
@@ -598,6 +973,24 @@ func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) {
}
})
t.Run("fallback profile source", func(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
promptkit.WithFallbackProfileFS(contractProfileFS("profile", "first-model"), "."),
promptkit.WithFallbackProfileFS(contractProfileFS("profile", "second-model"), "."),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatalf("prepare from last fallback profile source: %v", err)
}
if prepared.EffectiveModelParams.Model != "second-model" {
t.Fatalf("expected last fallback profile source, got %q", prepared.EffectiveModelParams.Model)
}
})
t.Run("in-memory profiles", func(t *testing.T) {
first := profile
first.Model = "first-model"
@@ -682,6 +1075,212 @@ func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) {
})
}
func TestFallbackProfileSourcePrecedence(t *testing.T) {
const profileID = "application-profile"
prepareModel := func(t *testing.T, engine *promptkit.Engine, promptID string) string {
t.Helper()
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: promptID})
if err != nil {
t.Fatalf("prepare: %v", err)
}
return prepared.EffectiveModelParams.Model
}
t.Run("in-memory profiles override ordinary and fallback profiles", func(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
promptkit.WithProfileFS(contractProfileFS(profileID, "ordinary-model"), "."),
promptkit.WithProfiles(promptkit.Profile{ID: profileID, Endpoint: "http://example.test/v1", Model: "memory-model"}),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
if model := prepareModel(t, engine, "prompt"); model != "memory-model" {
t.Fatalf("expected in-memory profile, got %q", model)
}
})
t.Run("ordinary filesystem source overrides fallback profile", func(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
promptkit.WithProfileFS(contractProfileFS(profileID, "ordinary-model"), "."),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
if model := prepareModel(t, engine, "prompt"); model != "ordinary-model" {
t.Fatalf("expected ordinary profile, got %q", model)
}
})
t.Run("configured directory overrides fallback profile", func(t *testing.T) {
profileDir := t.TempDir()
writePublicProfileFile(t, profileDir, profileID, "http://example.test/v1", "directory-model")
engine, err := promptkit.NewEngine(promptkit.Config{ProfileDir: profileDir},
promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
if model := prepareModel(t, engine, "prompt"); model != "directory-model" {
t.Fatalf("expected configured directory profile, got %q", model)
}
})
t.Run("fallback profile overrides built-in profile", func(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."),
promptkit.WithFallbackProfileFS(contractProfileFS("mistral-small-3", "fallback-model"), "."),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
if model := prepareModel(t, engine, "prompt"); model != "fallback-model" {
t.Fatalf("expected fallback profile, got %q", model)
}
})
t.Run("missing fallback profile uses built-in profile", func(t *testing.T) {
t.Setenv("OPENROUTER_API_KEY", "test-key")
baseline, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."),
)
if err != nil {
t.Fatalf("construct baseline engine: %v", err)
}
want := prepareModel(t, baseline, "prompt")
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."),
promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
if model := prepareModel(t, engine, "prompt"); model != want {
t.Fatalf("expected built-in profile model %q, got %q", want, model)
}
})
}
func TestFallbackProfileSourcePreservesLazyLoadingAndErrors(t *testing.T) {
const profileID = "application-profile"
t.Run("construction defers malformed fallback profiles", func(t *testing.T) {
_, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(fstest.MapFS{}, "."),
promptkit.WithFallbackProfileFS(fstest.MapFS{
"broken.yaml": &fstest.MapFile{Data: []byte("id: broken\nunknown: value\n")},
}, "."),
)
if err != nil {
t.Fatalf("construct engine with malformed fallback profile: %v", err)
}
})
t.Run("unrelated malformed fallback profile does not block matching definition", func(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
promptkit.WithFallbackProfileFS(fstest.MapFS{
"broken.yaml": &fstest.MapFile{Data: []byte("id: unrelated\nunknown: value\n")},
"valid.yaml": &fstest.MapFile{Data: []byte("id: application-profile\nendpoint: http://example.test/v1\nmodel: fallback-model\n")},
}, "."),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatalf("prepare from valid fallback profile: %v", err)
}
if prepared.EffectiveModelParams.Model != "fallback-model" {
t.Fatalf("unexpected fallback profile model: %q", prepared.EffectiveModelParams.Model)
}
})
t.Run("matching malformed fallback profile does not reach built-in profile", func(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."),
promptkit.WithFallbackProfileFS(fstest.MapFS{
"mistral-small-3.yaml": &fstest.MapFile{Data: []byte("id: mistral-small-3\nendpoint: http://example.test/v1\nmodel: fallback-model\nunknown: value\n")},
}, "."),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
if _, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"}); !errors.Is(err, promptkit.ErrProfileLoad) {
t.Fatalf("expected ErrProfileLoad, got %v", err)
}
})
t.Run("matching malformed ordinary profile does not reach fallback profile", func(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
promptkit.WithProfileFS(fstest.MapFS{
"application-profile.yaml": &fstest.MapFile{Data: []byte("id: application-profile\nendpoint: http://example.test/v1\nmodel: ordinary-model\nunknown: value\n")},
}, "."),
promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
if _, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"}); !errors.Is(err, promptkit.ErrProfileLoad) {
t.Fatalf("expected ErrProfileLoad, got %v", err)
}
})
}
func TestFallbackProfileSourceWorksAcrossWorkflows(t *testing.T) {
const profileID = "application-profile"
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
promptkit.WithLLMClient(client),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
inspection, err := engine.InspectProfile(context.Background(), profileID)
if err != nil {
t.Fatalf("inspect fallback profile: %v", err)
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatalf("prepare fallback profile: %v", err)
}
preparedExecution, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatalf("prepare execution with fallback profile: %v", err)
}
preparedDetails := preparedExecution.Details()
preparedResult, err := engine.RunPrepared(context.Background(), preparedExecution)
if err != nil {
t.Fatalf("run prepared fallback profile: %v", err)
}
runResult, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatalf("run fallback profile: %v", err)
}
for name, model := range map[string]string{
"inspection": inspection.EffectiveModelParams.Model,
"preparation": prepared.EffectiveModelParams.Model,
"prepared execution": preparedDetails.EffectiveModelParams.Model,
"prepared result": preparedResult.EffectiveModelParams.Model,
"run result": runResult.EffectiveModelParams.Model,
} {
if model != "fallback-model" {
t.Fatalf("%s model=%q, want fallback-model", name, model)
}
}
}
func TestEngineSupportsConcurrentPrepareAndRun(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),

123
types.go
View File

@@ -116,7 +116,8 @@ type RunRequest struct {
// empty maps are equivalent.
Vars map[string]string
// Execution optionally overrides individual execution settings. Nil uses
// the selected profile over its backend, when any, and framework defaults.
// the selected profile over its backend, when any, and the framework
// baseline.
Execution *ExecutionTargetOverride
// Validation optionally replaces the prompt's complete output contract. It
// does not merge individual fields. Nil uses the prompt contract.
@@ -144,9 +145,10 @@ type PreparedRun struct {
// SelectedBackendID equals EffectiveModelParams.BackendID. It is empty for
// an endpoint-only profile.
SelectedBackendID string `json:"selected_backend_id,omitempty"`
// EffectiveModelParams contains framework defaults overlaid by the selected
// backend, profile, and then request overrides. It excludes resolved API-key
// values.
// EffectiveModelParams contains settings resolved from the framework timeout
// baseline, selected backend, profile, and then request overrides. Unset
// optional provider controls remain zero rather than reporting a provider
// default. It excludes resolved API-key values.
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
// OutputContract is the complete effective output contract.
OutputContract OutputContract `json:"output_contract"`
@@ -298,13 +300,17 @@ type ExecutionTarget struct {
Endpoint string `json:"endpoint"`
// Model is the provider model identifier.
Model string `json:"model"`
// Temperature is the effective sampling temperature from 0 through 2.
// Temperature is the resolved sampling temperature from 0 through 2. Zero
// leaves the field unspecified to compatible providers unless the
// corresponding ExecutionTargetPresence bit is true.
Temperature float64 `json:"temperature"`
// MaxTokens is the non-negative effective output-token limit. Zero leaves
// the limit unspecified to compatible providers unless it was an explicit
// request override.
// MaxTokens is the non-negative resolved output-token limit. Zero leaves
// the limit unspecified to compatible providers unless the corresponding
// ExecutionTargetPresence bit is true.
MaxTokens int `json:"max_tokens"`
// TopP is the effective nucleus-sampling value from 0 through 1.
// TopP is the resolved nucleus-sampling value from 0 through 1. Zero leaves
// the field unspecified to compatible providers unless the corresponding
// ExecutionTargetPresence bit is true.
TopP float64 `json:"top_p"`
// TimeoutSeconds is the non-negative per-generation deadline. Zero disables
// this deadline without disabling caller cancellation or the transport cap.
@@ -320,6 +326,63 @@ type ExecutionTarget struct {
ExtraParams map[string]any `json:"extra_params"`
}
// ProfileInspection is the caller-owned result of [Engine.InspectProfile].
// It has no stable JSON representation.
//
// EffectiveModelParams contains a copied effective target. APIKeyRequired is
// separate from that target to preserve ExecutionTarget's general execution
// and stable JSON contracts.
type ProfileInspection struct {
// ProfileID is the trimmed, exact profile ID inspected by the engine.
ProfileID string
// EffectiveModelParams contains settings resolved from the framework timeout
// baseline, selected backend, and then profile, without a request override.
// Unset optional provider controls remain zero rather than reporting a
// provider default. APIKeyEnv is an environment-variable name, never its
// credential value.
EffectiveModelParams ExecutionTarget
// APIKeyRequired reports that a later execution must supply a direct API
// key or an explicit request environment override. It is mutually exclusive
// with a nonblank EffectiveModelParams.APIKeyEnv.
APIKeyRequired bool
}
// PromptInputDefinition describes one declared prompt input.
// It has no stable JSON representation.
type PromptInputDefinition struct {
// Name is the normalized prompt input name.
Name string
// Required is the prompt definition's declared required flag. When true,
// preparation fails if the input is omitted. A false value does not account
// for input references in message or session-ID templates.
Required bool
// ContentType is the declared input media-type metadata.
ContentType string
// Description is the declared human-readable input description.
Description string
}
// PromptInspection is the caller-owned result of [Engine.InspectPrompt].
// It has no stable JSON representation.
//
// Inputs contains copied declared input metadata in definition order.
// OutputContract is the normalized contract declared by the prompt definition,
// rather than a request-level effective override. PromptHash is opaque.
type PromptInspection struct {
// PromptID is the normalized ID of the selected prompt definition.
PromptID string
// PromptVersion is the normalized version of the selected prompt definition.
PromptVersion string
// PromptHash is the opaque equality value for the selected definition.
PromptHash string
// DefaultProfileID is declared metadata and is not resolved by inspection.
DefaultProfileID string
// Inputs contains caller-owned declared input metadata in definition order.
Inputs []PromptInputDefinition
// OutputContract is the normalized contract declared by the definition.
OutputContract OutputContract
}
// ExecutionTargetOverride represents per-request runtime setting overrides and
// has no stable JSON representation.
//
@@ -327,19 +390,26 @@ type ExecutionTarget struct {
// fields replace profile values and preserve explicit zero or empty values. A
// non-empty ExtraParams map replaces the complete profile or backend map
// rather than merging keys. Empty string fields, nil pointers, and a nil or
// empty ExtraParams map inherit the selected profile over its backend, when
// any, and framework defaults.
// empty ExtraParams map inherit lower-precedence values. An optional provider
// control that remains zero is unspecified; TimeoutSeconds retains its
// framework deadline when no higher-precedence value is present.
type ExecutionTargetOverride struct {
// Endpoint replaces the profile or backend endpoint when non-empty without
// changing the effective BackendID.
Endpoint string
// Model replaces the profile model when non-empty.
Model string
// Temperature, when non-nil, must point to a value from 0 through 2.
// Temperature, when non-nil, must point to a value from 0 through 2. A
// pointed-to zero is explicitly present; nil inherits a lower-precedence
// value and otherwise leaves the provider control unspecified.
Temperature *float64
// MaxTokens, when non-nil, must point to a non-negative value.
// MaxTokens, when non-nil, must point to a non-negative value. A pointed-to
// zero is explicitly present; nil inherits a lower-precedence value and
// otherwise leaves the provider control unspecified.
MaxTokens *int
// TopP, when non-nil, must point to a value from 0 through 1.
// TopP, when non-nil, must point to a value from 0 through 1. A pointed-to
// zero is explicitly present; nil inherits a lower-precedence value and
// otherwise leaves the provider control unspecified.
TopP *float64
// TimeoutSeconds, when non-nil, must point to a non-negative value. A
// pointed-to zero disables the per-generation deadline.
@@ -370,9 +440,12 @@ type ExecutionTargetOverride struct {
// use profile YAML api_key_env with file and FS profile sources. Profile has no
// stable JSON representation.
//
// WithProfiles validates and copies Profile values during NewEngine. Numeric
// zero, blank strings, and an empty ExtraParams map inherit framework defaults;
// use ExecutionTargetOverride pointer fields to request explicit numeric zero.
// WithProfiles validates and copies Profile values during NewEngine. Zero
// Temperature, MaxTokens, and TopP values and blank ServiceTier and
// ReasoningEffort values leave those provider controls unspecified. A zero
// TimeoutSeconds retains the framework deadline, while an empty ExtraParams map
// inherits backend request defaults. Use ExecutionTargetOverride pointer fields
// to request an explicit numeric zero.
type Profile struct {
// ID is the required non-blank profile identifier. WithProfiles trims it.
ID string
@@ -386,19 +459,19 @@ type Profile struct {
Endpoint string
// Model is the required non-blank provider model identifier.
Model string
// Temperature is from 0 through 2. Zero inherits the framework default.
// Temperature is from 0 through 2. Zero leaves the provider control
// unspecified.
Temperature float64
// MaxTokens is non-negative. Zero inherits the framework default.
// MaxTokens is non-negative. Zero leaves the provider control unspecified.
MaxTokens int
// TopP is from 0 through 1. Zero inherits the framework default rather than
// selecting an explicit zero.
// TopP is from 0 through 1. Zero leaves the provider control unspecified
// rather than selecting an explicit zero.
TopP float64
// TimeoutSeconds is non-negative. Zero inherits the framework default.
// TimeoutSeconds is non-negative. Zero retains the framework deadline.
TimeoutSeconds int
// ServiceTier is optional; a blank value inherits the framework default.
// ServiceTier is optional; a blank value leaves it unspecified.
ServiceTier string
// ReasoningEffort is optional; a blank value inherits the framework
// default.
// ReasoningEffort is optional; a blank value leaves it unspecified.
ReasoningEffort string
// APIKeyRequired clears a backend's inherited API-key environment name and
// requires a non-blank RunRequest.APIKey unless the request explicitly