Compare commits
6 Commits
v0.2.0
...
c13e9710d9
| Author | SHA1 | Date | |
|---|---|---|---|
| c13e9710d9 | |||
| 87b5ec3d75 | |||
| cb4028a637 | |||
| 5a1bff4529 | |||
| 805a7f965d | |||
| 147f5e5ff5 |
@@ -33,6 +33,9 @@ boundary and constraints that framework work must preserve.
|
|||||||
|
|
||||||
## Release Guidance
|
## Release Guidance
|
||||||
|
|
||||||
|
Consumers upgrading from `v0.2.0` to `v0.3.0` should read the
|
||||||
|
[v0.3.0 changelog](docs/releases/v0.3.0.md).
|
||||||
|
|
||||||
Consumers moving from `v0.1.0` to `v0.2.0` should read the
|
Consumers moving from `v0.1.0` to `v0.2.0` should read the
|
||||||
[v0.2.0 changelog and migration guide](docs/releases/v0.2.0.md).
|
[v0.2.0 changelog and migration guide](docs/releases/v0.2.0.md).
|
||||||
|
|
||||||
|
|||||||
25
backends.go
25
backends.go
@@ -9,6 +9,11 @@ import (
|
|||||||
// backend.
|
// backend.
|
||||||
const BackendOpenRouter = backend.OpenRouterID
|
const BackendOpenRouter = backend.OpenRouterID
|
||||||
|
|
||||||
|
// BackendLocal is the case-sensitive conventional ID used by [LocalBackend].
|
||||||
|
// It is not a built-in or reserved backend and must be registered with
|
||||||
|
// [WithBackend].
|
||||||
|
const BackendLocal = "local"
|
||||||
|
|
||||||
// Backend configures one engine-scoped OpenAI-compatible backend.
|
// Backend configures one engine-scoped OpenAI-compatible backend.
|
||||||
//
|
//
|
||||||
// Backend has no stable JSON representation. Use keyed literals so additions
|
// Backend has no stable JSON representation. Use keyed literals so additions
|
||||||
@@ -44,6 +49,26 @@ type Backend struct {
|
|||||||
QueueCapacity *int
|
QueueCapacity *int
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// LocalBackend returns a caller-owned Backend for a conventional local
|
||||||
|
// OpenAI-compatible endpoint. It sets ID to BackendLocal and copies endpoint
|
||||||
|
// and concurrencyLimit into Endpoint and ConcurrencyLimit without
|
||||||
|
// normalization or validation. APIKeyEnv, ExtraParams, and QueueCapacity keep
|
||||||
|
// their zero values.
|
||||||
|
//
|
||||||
|
// LocalBackend does not read environment variables, register the value, or
|
||||||
|
// mutate engine or package state. Supply the returned value through
|
||||||
|
// [WithBackend]; [NewEngine] then applies the ordinary backend validation and
|
||||||
|
// concurrency semantics, including default queue capacity for a positive
|
||||||
|
// limit, unlimited behavior for zero, and ErrInvalidConfig for a negative
|
||||||
|
// limit.
|
||||||
|
func LocalBackend(endpoint string, concurrencyLimit int) Backend {
|
||||||
|
return Backend{
|
||||||
|
ID: BackendLocal,
|
||||||
|
Endpoint: endpoint,
|
||||||
|
ConcurrencyLimit: concurrencyLimit,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// WithBackend adds one Backend registration to the constructed Engine.
|
// WithBackend adds one Backend registration to the constructed Engine.
|
||||||
//
|
//
|
||||||
// Registrations accumulate in option order. Every normalized ID must be unique
|
// Registrations accumulate in option order. Every normalized ID must be unique
|
||||||
|
|||||||
@@ -124,35 +124,86 @@ providers. The
|
|||||||
[`RunRequest` and `ExecutionTargetOverride` GoDoc](../../types.go) owns the
|
[`RunRequest` and `ExecutionTargetOverride` GoDoc](../../types.go) owns the
|
||||||
exact normalization, precedence, error, copying, and exposure contract.
|
exact normalization, precedence, error, copying, and exposure contract.
|
||||||
|
|
||||||
### Register A Custom Backend
|
### Configure A Local OpenAI-Compatible Endpoint
|
||||||
|
|
||||||
Register a reusable OpenAI-compatible connection once, then select it from a
|
Choose the smallest configuration that fits how the endpoint will be reused.
|
||||||
profile. This local backend limits model generation to two simultaneous calls;
|
|
||||||
because `QueueCapacity` is omitted, the engine admits up to 1024 additional
|
#### Use An Endpoint-Only Profile
|
||||||
calls waiting behind them:
|
|
||||||
|
Put the endpoint directly on an in-memory profile when only that profile needs
|
||||||
|
it and shared backend identity or capacity policy is unnecessary:
|
||||||
|
|
||||||
```go
|
```go
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
PromptDir: "prompts",
|
PromptDir: "prompts",
|
||||||
},
|
},
|
||||||
promptkit.WithBackend(promptkit.Backend{
|
|
||||||
ID: "local",
|
|
||||||
Endpoint: "http://localhost:8000/v1",
|
|
||||||
APIKeyEnv: "LOCAL_LLM_API_KEY",
|
|
||||||
ConcurrencyLimit: 2,
|
|
||||||
}),
|
|
||||||
promptkit.WithProfiles(promptkit.Profile{
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
ID: "local-summary",
|
ID: "local-summary",
|
||||||
BackendID: "local",
|
Endpoint: "http://localhost:8000/v1",
|
||||||
Model: "example-model",
|
Model: "example-model",
|
||||||
}),
|
}),
|
||||||
)
|
)
|
||||||
```
|
```
|
||||||
|
|
||||||
|
Endpoint-only profiles have an empty backend ID and remain unrestricted by
|
||||||
|
backend capacity policy.
|
||||||
|
|
||||||
|
#### Use The Conventional Local Backend
|
||||||
|
|
||||||
|
Use `LocalBackend` when profiles should share the conventional `local`
|
||||||
|
identity, endpoint, and concurrency limit:
|
||||||
|
|
||||||
|
```go
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
|
PromptDir: "prompts",
|
||||||
|
},
|
||||||
|
promptkit.WithBackend(
|
||||||
|
promptkit.LocalBackend("http://localhost:8000/v1", 2),
|
||||||
|
),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "local-summary",
|
||||||
|
BackendID: promptkit.BackendLocal,
|
||||||
|
Model: "example-model",
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
The helper is explicit: it does not pre-register a backend or read environment
|
||||||
|
variables. Supplying a positive limit leaves queue capacity omitted, so normal
|
||||||
|
backend registration selects the existing default waiting capacity of 1024.
|
||||||
|
The returned value still enters the engine through `WithBackend`.
|
||||||
|
|
||||||
|
#### Configure A Complete Backend
|
||||||
|
|
||||||
|
Use a keyed `Backend` value for authentication, extra request parameters, an
|
||||||
|
explicit queue capacity, a custom ID, or multiple local endpoints:
|
||||||
|
|
||||||
|
```go
|
||||||
|
noWaiting := 0
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
|
PromptDir: "prompts",
|
||||||
|
},
|
||||||
|
promptkit.WithBackend(promptkit.Backend{
|
||||||
|
ID: "local-gpu",
|
||||||
|
Endpoint: "http://gpu-host:8000/v1",
|
||||||
|
APIKeyEnv: "LOCAL_GPU_API_KEY",
|
||||||
|
ExtraParams: map[string]any{"provider_option": "enabled"},
|
||||||
|
ConcurrencyLimit: 2,
|
||||||
|
QueueCapacity: &noWaiting,
|
||||||
|
}),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "gpu-summary",
|
||||||
|
BackendID: "local-gpu",
|
||||||
|
Model: "example-model",
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
Use distinct custom IDs when registering multiple local endpoints.
|
||||||
Registrations belong to one engine and custom IDs cannot replace built-ins.
|
Registrations belong to one engine and custom IDs cannot replace built-ins.
|
||||||
The [`Backend` and `WithBackend` GoDoc](../../backends.go) defines validation,
|
The [`Backend`, `LocalBackend`, and `WithBackend` GoDoc](../../backends.go)
|
||||||
copying, uniqueness, exact concurrency-field semantics, and request-default
|
defines exact construction, validation, copying, uniqueness, concurrency, and
|
||||||
behavior.
|
request-default behavior.
|
||||||
|
|
||||||
Both file-backed and in-memory profiles select a registration through
|
Both file-backed and in-memory profiles select a registration through
|
||||||
`backend` or `Profile.BackendID`. Profile and request endpoint overrides retain
|
`backend` or `Profile.BackendID`. Profile and request endpoint overrides retain
|
||||||
@@ -173,8 +224,8 @@ zero:
|
|||||||
```go
|
```go
|
||||||
noWaiting := 0
|
noWaiting := 0
|
||||||
backend := promptkit.Backend{
|
backend := promptkit.Backend{
|
||||||
ID: "local",
|
ID: "local-gpu",
|
||||||
Endpoint: "http://localhost:8000/v1",
|
Endpoint: "http://gpu-host:8000/v1",
|
||||||
ConcurrencyLimit: 2,
|
ConcurrencyLimit: 2,
|
||||||
QueueCapacity: &noWaiting,
|
QueueCapacity: &noWaiting,
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -90,7 +90,7 @@ engine, err := promptkit.NewEngine(
|
|||||||
```
|
```
|
||||||
|
|
||||||
See the
|
See the
|
||||||
[custom-backend consumer guide](../consumers/pkg-promptkit.md#register-a-custom-backend)
|
[local-endpoint consumer guide](../consumers/pkg-promptkit.md#configure-a-local-openai-compatible-endpoint)
|
||||||
for task-oriented usage. The
|
for task-oriented usage. The
|
||||||
[`Backend` and `WithBackend` GoDoc](../../backends.go) owns exact registration,
|
[`Backend` and `WithBackend` GoDoc](../../backends.go) owns exact registration,
|
||||||
validation, copying, defaulting, and uniqueness semantics. The
|
validation, copying, defaulting, and uniqueness semantics. The
|
||||||
|
|||||||
74
docs/releases/v0.3.0.md
Normal file
74
docs/releases/v0.3.0.md
Normal file
@@ -0,0 +1,74 @@
|
|||||||
|
# Promptkit v0.3.0
|
||||||
|
|
||||||
|
This supplemental changelog summarizes the consumer-facing changes from
|
||||||
|
`v0.2.0` to `v0.3.0`. The annotated `v0.3.0` tag is the authoritative release
|
||||||
|
record. Exact current contracts belong to the linked GoDoc and durable
|
||||||
|
documentation.
|
||||||
|
|
||||||
|
## Summary
|
||||||
|
|
||||||
|
`v0.3.0` adds a concise way to register the common local OpenAI-compatible
|
||||||
|
backend configuration:
|
||||||
|
|
||||||
|
- `BackendLocal` provides the conventional, non-reserved backend ID `"local"`;
|
||||||
|
and
|
||||||
|
- `LocalBackend` constructs an ordinary `Backend` from an endpoint and
|
||||||
|
concurrency limit.
|
||||||
|
|
||||||
|
The helper is explicit and additive. It does not pre-register a backend, read
|
||||||
|
environment variables, select a model, or replace the complete `Backend`
|
||||||
|
configuration interface.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
Existing `v0.2.0` consumers require no migration. Endpoint-only profiles,
|
||||||
|
complete custom `Backend` values, the built-in OpenRouter backend, and existing
|
||||||
|
registrations using the literal ID `"local"` continue to work unchanged.
|
||||||
|
|
||||||
|
## Upgrade
|
||||||
|
|
||||||
|
Update the module dependency with:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go get gitea.maximumdirect.net/eric/promptkit@v0.3.0
|
||||||
|
go mod tidy
|
||||||
|
```
|
||||||
|
|
||||||
|
Run the consuming project's ordinary tests and race-enabled tests after the
|
||||||
|
upgrade.
|
||||||
|
|
||||||
|
## Configure A Local Backend
|
||||||
|
|
||||||
|
Register the convenience value through the existing `WithBackend` option and
|
||||||
|
select it from one or more profiles:
|
||||||
|
|
||||||
|
```go
|
||||||
|
engine, err := promptkit.NewEngine(
|
||||||
|
promptkit.Config{PromptDir: "prompts"},
|
||||||
|
promptkit.WithBackend(
|
||||||
|
promptkit.LocalBackend("http://localhost:8000/v1", 2),
|
||||||
|
),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "local-summary",
|
||||||
|
BackendID: promptkit.BackendLocal,
|
||||||
|
Model: "example-model",
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
Use an endpoint-only profile when shared backend identity and capacity policy
|
||||||
|
are unnecessary. Continue to use a complete keyed `Backend` value for custom
|
||||||
|
IDs, authentication, extra request parameters, explicit queue capacity, or
|
||||||
|
multiple local endpoints.
|
||||||
|
|
||||||
|
See the
|
||||||
|
[local-endpoint consumer guide](../consumers/pkg-promptkit.md#configure-a-local-openai-compatible-endpoint)
|
||||||
|
for task-oriented configuration choices. The
|
||||||
|
[`BackendLocal`, `LocalBackend`, and `WithBackend` GoDoc](../../backends.go)
|
||||||
|
owns their exact construction, registration, validation, and concurrency
|
||||||
|
semantics.
|
||||||
|
|
||||||
|
## Consumer Action
|
||||||
|
|
||||||
|
None. Adopt the convenience constructor when it simplifies local endpoint
|
||||||
|
configuration.
|
||||||
@@ -11,7 +11,8 @@ consumer value, and important policy choices.
|
|||||||
This document is planning material, not a description of current behavior.
|
This document is planning material, not a description of current behavior.
|
||||||
Current exported contracts remain owned by Go declarations and GoDoc, backend
|
Current exported contracts remain owned by Go declarations and GoDoc, backend
|
||||||
registration guidance by the
|
registration guidance by the
|
||||||
[consumer guide](../consumers/pkg-promptkit.md#register-a-custom-backend), and
|
[consumer guide](../consumers/pkg-promptkit.md#configure-a-local-openai-compatible-endpoint),
|
||||||
|
and
|
||||||
implemented orchestration by the
|
implemented orchestration by the
|
||||||
[internal runner document](../internal/runner.md).
|
[internal runner document](../internal/runner.md).
|
||||||
|
|
||||||
|
|||||||
@@ -33,9 +33,52 @@ consumers.
|
|||||||
|
|
||||||
## Ideas
|
## Ideas
|
||||||
|
|
||||||
No ideas are currently cataloged. Backend-specific concurrency management has
|
Executable preparation handles have been selected for active planning in the
|
||||||
been selected for active planning in the
|
[focused feature roadmap](prepared-execution.md). The remaining ideas are
|
||||||
[focused concurrency roadmap](concurrency.md).
|
still available for future selection.
|
||||||
|
|
||||||
|
### Prompt-independent profile inspection
|
||||||
|
|
||||||
|
Provide exact profile lookup and structural resolution without requiring a
|
||||||
|
synthetic prompt, placeholder inputs, or model generation. This shared need is
|
||||||
|
described by
|
||||||
|
[Notarius](notarius-promptkit-wishlist.md#priority-2-prompt-independent-profile-inspection)
|
||||||
|
and
|
||||||
|
[Weatherreporter](weatherreporter-promptkit-wishlist.md#priority-3-prompt-independent-profile-inspection).
|
||||||
|
|
||||||
|
- Apply ordinary built-in, file-backed, and programmatic profile precedence.
|
||||||
|
- Validate referenced backend membership and the structurally resolved
|
||||||
|
execution target.
|
||||||
|
- Report credential requirements and environment-variable names without
|
||||||
|
exposing credential values or requiring current credential availability.
|
||||||
|
- Support exact lookup by profile ID; enumeration is not required initially.
|
||||||
|
|
||||||
|
### Prompt-definition inspection
|
||||||
|
|
||||||
|
Provide exact prompt-definition lookup without rendering, placeholder inputs,
|
||||||
|
profile resolution, or model generation, as requested by
|
||||||
|
[Weatherreporter](weatherreporter-promptkit-wishlist.md#priority-2-prompt-definition-inspection).
|
||||||
|
|
||||||
|
- Return caller-owned identity, version, input definitions, default-profile,
|
||||||
|
output-contract, and opaque definition-equality information.
|
||||||
|
- Apply ordinary prompt-source precedence and exact ID/version selection.
|
||||||
|
- Validate the selected definition and referenced prompt content
|
||||||
|
structurally, without returning source bodies or rendered messages.
|
||||||
|
- Leave complete cross-source corpus validation and enumeration outside the
|
||||||
|
initial inspection contract.
|
||||||
|
|
||||||
|
### Structured capacity errors
|
||||||
|
|
||||||
|
Add safe structured context to backend admission rejection, as requested by
|
||||||
|
[Notarius](notarius-promptkit-wishlist.md#priority-4-structured-capacity-errors)
|
||||||
|
and
|
||||||
|
[Weatherreporter](weatherreporter-promptkit-wishlist.md#structured-capacity-errors).
|
||||||
|
|
||||||
|
- Preserve compatibility with `errors.Is(err, ErrCapacityExceeded)`.
|
||||||
|
- Support `errors.As` to obtain the stable backend ID.
|
||||||
|
- Do not expose endpoints, credential configuration or values, request
|
||||||
|
content, or speculative retry timing.
|
||||||
|
- Keep retry and backoff policy with downstream consumers.
|
||||||
|
|
||||||
## Entry Format
|
## Entry Format
|
||||||
|
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
163
docs/roadmap/local-backend.md
Normal file
163
docs/roadmap/local-backend.md
Normal file
@@ -0,0 +1,163 @@
|
|||||||
|
# Local Backend Convenience
|
||||||
|
|
||||||
|
**Status:** Complete.
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Make the common case of using a local OpenAI-compatible endpoint concise and
|
||||||
|
easy to discover without introducing implicit configuration or a separate
|
||||||
|
backend abstraction.
|
||||||
|
|
||||||
|
The existing `Backend` type and registry remain the canonical, fully
|
||||||
|
configurable interface. A small convenience constructor will cover the usual
|
||||||
|
local-network case, while improved consumer documentation will make it clear
|
||||||
|
when an endpoint-only profile, the convenience constructor, or a complete
|
||||||
|
`Backend` value is appropriate.
|
||||||
|
|
||||||
|
## Motivation
|
||||||
|
|
||||||
|
Consumers can already use a local endpoint by setting `Profile.Endpoint`, or
|
||||||
|
register one as a backend with `WithBackend`. The first option is concise but
|
||||||
|
does not provide shared backend-level concurrency control. The second supports
|
||||||
|
the complete backend feature set but requires consumers to understand and
|
||||||
|
populate several fields for a common configuration.
|
||||||
|
|
||||||
|
Most consumers adding a local backend need only:
|
||||||
|
|
||||||
|
- a stable backend ID;
|
||||||
|
- an OpenAI-compatible endpoint; and
|
||||||
|
- a concurrency limit appropriate for the local server.
|
||||||
|
|
||||||
|
Promptkit should provide a direct path for that case while keeping all
|
||||||
|
configuration explicit and preserving the full registry interface for
|
||||||
|
advanced needs.
|
||||||
|
|
||||||
|
## Consumer Paths
|
||||||
|
|
||||||
|
Documentation should present three progressively more configurable paths:
|
||||||
|
|
||||||
|
1. Set `Profile.Endpoint` when a profile only needs to target a local endpoint
|
||||||
|
and does not need shared backend policy.
|
||||||
|
2. Use the local-backend convenience constructor when profiles should share a
|
||||||
|
named local endpoint and its concurrency limit.
|
||||||
|
3. Construct a complete `Backend` value when the consumer needs a custom
|
||||||
|
backend ID, authentication, extra request parameters, an explicit queue
|
||||||
|
capacity, or multiple local backends.
|
||||||
|
|
||||||
|
These are complementary interfaces. The convenience constructor must return an
|
||||||
|
ordinary `Backend`, so it does not create a second configuration model.
|
||||||
|
|
||||||
|
## Public Convenience API
|
||||||
|
|
||||||
|
The public package should expose:
|
||||||
|
|
||||||
|
```go
|
||||||
|
const BackendLocal = "local"
|
||||||
|
|
||||||
|
func LocalBackend(endpoint string, concurrencyLimit int) Backend
|
||||||
|
```
|
||||||
|
|
||||||
|
`LocalBackend` should return a `Backend` with:
|
||||||
|
|
||||||
|
- `ID` set to `BackendLocal`;
|
||||||
|
- `Endpoint` set to the supplied endpoint;
|
||||||
|
- `ConcurrencyLimit` set to the supplied limit; and
|
||||||
|
- all other fields left at their zero values.
|
||||||
|
|
||||||
|
The returned value is passed to `WithBackend` and follows the same copying,
|
||||||
|
normalization, validation, and registration rules as any consumer-constructed
|
||||||
|
`Backend`.
|
||||||
|
|
||||||
|
The constructor should be a transparent value constructor. It should not read
|
||||||
|
environment variables, mutate global state, register the backend, validate
|
||||||
|
arguments independently, or create profiles. Consumers may inspect or modify
|
||||||
|
the returned value before registration, although documentation should direct
|
||||||
|
substantially customized configurations to the full `Backend` form.
|
||||||
|
|
||||||
|
## Identity and Registration
|
||||||
|
|
||||||
|
`BackendLocal` is a conventional ID used by the convenience constructor. It is
|
||||||
|
not pre-registered and should not become a specially reserved registry ID.
|
||||||
|
Consumers remain responsible for registering the returned backend with
|
||||||
|
`WithBackend` and naming it from profiles through `BackendID`.
|
||||||
|
|
||||||
|
This distinction preserves compatibility with consumers that may already
|
||||||
|
register their own backend using the ID `"local"`. Normal duplicate-ID rules
|
||||||
|
still apply if a consumer attempts to register more than one backend with that
|
||||||
|
ID.
|
||||||
|
|
||||||
|
Consumers that need multiple local endpoints should choose distinct IDs and
|
||||||
|
use complete `Backend` values rather than the single conventional helper ID.
|
||||||
|
|
||||||
|
## Concurrency and Queue Semantics
|
||||||
|
|
||||||
|
The constructor must preserve the existing backend concurrency contract:
|
||||||
|
|
||||||
|
- a positive concurrency limit bounds simultaneous requests and uses the
|
||||||
|
existing default queue capacity because `QueueCapacity` remains `nil`;
|
||||||
|
- a zero concurrency limit leaves the backend unconstrained; and
|
||||||
|
- a negative concurrency limit is rejected through the existing engine
|
||||||
|
configuration validation path.
|
||||||
|
|
||||||
|
The constructor should not select a hidden default concurrency limit. Local
|
||||||
|
servers vary substantially in capacity, so the consumer should make this
|
||||||
|
choice explicitly.
|
||||||
|
|
||||||
|
## Documentation
|
||||||
|
|
||||||
|
The final documentation state has two canonical surfaces:
|
||||||
|
|
||||||
|
- Public Go documentation describes the exact contract of
|
||||||
|
`BackendLocal` and `LocalBackend`, including their conventional,
|
||||||
|
non-pre-registered nature.
|
||||||
|
- The [promptkit consumer guide](../consumers/pkg-promptkit.md) includes
|
||||||
|
a task-oriented local-endpoint section that shows the three consumer paths,
|
||||||
|
explains the decision between them, and provides concise examples of the
|
||||||
|
endpoint-only and convenience-constructor forms.
|
||||||
|
|
||||||
|
The consumer guide continues to document the full `Backend` interface as
|
||||||
|
the advanced path rather than attempting to reproduce every configuration
|
||||||
|
variation through convenience APIs.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
This feature is additive:
|
||||||
|
|
||||||
|
- existing endpoint-only profiles continue to work unchanged;
|
||||||
|
- existing `Backend` values and `WithBackend` registrations remain the
|
||||||
|
canonical general-purpose interface;
|
||||||
|
- existing registrations using the literal ID `"local"` remain valid; and
|
||||||
|
- OpenRouter defaults and all other backend behavior remain unchanged.
|
||||||
|
|
||||||
|
No consumer is required to adopt the convenience constructor.
|
||||||
|
|
||||||
|
## Non-Goals
|
||||||
|
|
||||||
|
This work does not include:
|
||||||
|
|
||||||
|
- pre-registering or implicitly enabling a local backend;
|
||||||
|
- discovering a local endpoint, API key, or concurrency limit from environment
|
||||||
|
variables;
|
||||||
|
- adding local-backend fields to `Config`;
|
||||||
|
- selecting a default local model or creating a profile automatically;
|
||||||
|
- adding a combined backend-and-profile constructor;
|
||||||
|
- adding convenience parameters for API keys, extra request parameters, or
|
||||||
|
queue capacity;
|
||||||
|
- replacing or redesigning the backend registry;
|
||||||
|
- adding support for non-OpenAI-compatible local APIs; or
|
||||||
|
- changing backend routing, scheduling, or queue behavior.
|
||||||
|
|
||||||
|
## Target End State
|
||||||
|
|
||||||
|
After this work:
|
||||||
|
|
||||||
|
- consumers with a simple one-profile local endpoint can continue to configure
|
||||||
|
it directly on the profile;
|
||||||
|
- consumers needing a shared local endpoint and concurrency policy can express
|
||||||
|
it with one `LocalBackend` call and register the returned value normally;
|
||||||
|
- consumers with advanced or multiple-local-backend requirements have a clear
|
||||||
|
path to the complete `Backend` interface;
|
||||||
|
- all local configuration remains explicit, inspectable, and compatible with
|
||||||
|
dependency injection; and
|
||||||
|
- canonical documentation makes the simplest suitable interface easy to find
|
||||||
|
without obscuring the underlying registry model.
|
||||||
358
docs/roadmap/notarius-promptkit-wishlist.md
Normal file
358
docs/roadmap/notarius-promptkit-wishlist.md
Normal file
@@ -0,0 +1,358 @@
|
|||||||
|
# Notarius PromptKit Wishlist
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
This document records features and interface changes that would be useful
|
||||||
|
additions to PromptKit from the perspective of the maintainers of Notarius, a
|
||||||
|
downstream application that consumes PromptKit.
|
||||||
|
|
||||||
|
PromptKit v0.3.0 provides the capabilities Notarius currently needs. None of
|
||||||
|
the ideas below blocks current Notarius development. They are opportunities to
|
||||||
|
reduce downstream workarounds, improve integration correctness, and make
|
||||||
|
PromptKit more ergonomic for applications with configuration validation,
|
||||||
|
debugging, checkpointing, and operational-observability requirements.
|
||||||
|
|
||||||
|
The examples are API sketches intended to communicate the desired capability,
|
||||||
|
not prescriptive names or finalized Go contracts.
|
||||||
|
|
||||||
|
## Priority 1: Atomic Execution With Prepared Details
|
||||||
|
|
||||||
|
**Disposition:** Covered by the accepted
|
||||||
|
[executable preparation handles](prepared-execution.md) roadmap. The shared
|
||||||
|
two-phase capability should provide the required single-preparation
|
||||||
|
consistency; a separate `RunDetailed` method is not cataloged initially.
|
||||||
|
|
||||||
|
### Downstream need
|
||||||
|
|
||||||
|
Notarius needs both:
|
||||||
|
|
||||||
|
- the completed `RunResult`; and
|
||||||
|
- the rendered messages, effective output contract, hashes, and other
|
||||||
|
preparation details exposed by `PreparedRun`.
|
||||||
|
|
||||||
|
Notarius uses the prepared details to construct redaction-aware debug bundles
|
||||||
|
and retain enough information to diagnose model behavior.
|
||||||
|
|
||||||
|
### Current integration
|
||||||
|
|
||||||
|
Notarius currently calls `Engine.Prepare` and then `Engine.Run` with the same
|
||||||
|
request. Because `Run` performs preparation internally, a successful request
|
||||||
|
resolves and prepares the same work twice.
|
||||||
|
|
||||||
|
This duplicates profile resolution, input hashing, schema loading, and prompt
|
||||||
|
rendering. It also creates a theoretical consistency window in which a
|
||||||
|
filesystem-backed prompt, profile, schema, or input could change between the
|
||||||
|
explicit preparation and the preparation performed by `Run`.
|
||||||
|
|
||||||
|
### Requested capability
|
||||||
|
|
||||||
|
Add an opt-in execution method that prepares exactly once and returns both the
|
||||||
|
prepared details and completed result:
|
||||||
|
|
||||||
|
```go
|
||||||
|
type RunReport struct {
|
||||||
|
Prepared PreparedRun
|
||||||
|
Result RunResult
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) RunDetailed(
|
||||||
|
ctx context.Context,
|
||||||
|
req RunRequest,
|
||||||
|
) (*RunReport, error)
|
||||||
|
```
|
||||||
|
|
||||||
|
The exact names are flexible. The important contract is that preparation
|
||||||
|
occurs once and that the returned prepared state describes the execution that
|
||||||
|
produced the returned result.
|
||||||
|
|
||||||
|
Existing `Prepare` and `Run` behavior should remain available for consumers
|
||||||
|
that need only one side of the operation.
|
||||||
|
|
||||||
|
### Design considerations
|
||||||
|
|
||||||
|
- Keep this API additive and preserve the existing simple `Run` workflow.
|
||||||
|
- Return caller-owned copies under PromptKit's existing ownership rules.
|
||||||
|
- Define whether any prepared details are available after an operational
|
||||||
|
generation or validation error. Notarius does not require partial results
|
||||||
|
for the initial use case, but an explicit contract would be valuable.
|
||||||
|
- Preserve cancellation and backend-admission semantics.
|
||||||
|
- Do not add all prepared content directly to `RunResult`. Rendered prompt
|
||||||
|
content can be large and sensitive, and consumers should opt in to receiving
|
||||||
|
it.
|
||||||
|
|
||||||
|
### Value to Notarius
|
||||||
|
|
||||||
|
This is the highest-value wishlist item. It would remove duplicate work from
|
||||||
|
every successful PromptKit-backed call and ensure that retained debug material
|
||||||
|
corresponds atomically to the actual execution.
|
||||||
|
|
||||||
|
## Priority 2: Prompt-Independent Profile Inspection
|
||||||
|
|
||||||
|
**Disposition:** Accepted into the
|
||||||
|
[future catalog](future.md#prompt-independent-profile-inspection).
|
||||||
|
|
||||||
|
### Downstream need
|
||||||
|
|
||||||
|
Notarius validates configured pipeline profile IDs before beginning a run. It
|
||||||
|
needs to determine whether:
|
||||||
|
|
||||||
|
- a profile exists;
|
||||||
|
- its referenced backend is registered;
|
||||||
|
- its execution target can be resolved; and
|
||||||
|
- it declares a credential requirement that the application may need to
|
||||||
|
enforce.
|
||||||
|
|
||||||
|
This validation should not require model generation.
|
||||||
|
|
||||||
|
### Current integration
|
||||||
|
|
||||||
|
Notarius constructs a synthetic prompt using `testing/fstest.MapFS`, supplies a
|
||||||
|
dummy transcript, and calls `Engine.Prepare` solely to exercise profile and
|
||||||
|
backend resolution. This works, but prompt preparation is serving as a
|
||||||
|
substitute for a profile-inspection interface.
|
||||||
|
|
||||||
|
### Requested capability
|
||||||
|
|
||||||
|
Add a prompt-independent profile-resolution API, for example:
|
||||||
|
|
||||||
|
```go
|
||||||
|
type ResolvedProfile struct {
|
||||||
|
ProfileID string
|
||||||
|
BackendID string
|
||||||
|
EffectiveTarget ExecutionTarget
|
||||||
|
APIKeyEnv string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) ResolveProfile(
|
||||||
|
ctx context.Context,
|
||||||
|
profileID string,
|
||||||
|
) (ResolvedProfile, error)
|
||||||
|
```
|
||||||
|
|
||||||
|
The returned shape may differ, but it should provide enough information for a
|
||||||
|
consumer to validate an explicit profile selection without inventing a prompt
|
||||||
|
or supplying placeholder inputs.
|
||||||
|
|
||||||
|
### Design considerations
|
||||||
|
|
||||||
|
- Resolve built-in, file-backed, and programmatic profiles using normal
|
||||||
|
PromptKit precedence.
|
||||||
|
- Validate that a referenced backend registration exists.
|
||||||
|
- Do not resolve, retain, or expose credential values.
|
||||||
|
- Report credential requirements, such as an environment-variable name, so
|
||||||
|
the consuming application can decide whether availability is required at
|
||||||
|
configuration-validation time or only at execution time.
|
||||||
|
- Return caller-owned values.
|
||||||
|
- Preserve typed or sentinel error classification for missing and invalid
|
||||||
|
profiles.
|
||||||
|
- Consider accepting an `ExecutionTargetOverride` if consumers need to inspect
|
||||||
|
the same effective target that a run-level override would produce.
|
||||||
|
- Enumeration of all profiles is not required for the Notarius use case; exact
|
||||||
|
lookup by ID is sufficient.
|
||||||
|
|
||||||
|
### Value to Notarius
|
||||||
|
|
||||||
|
This would eliminate a synthetic production-only prompt fixture and establish
|
||||||
|
a direct, supported contract for configuration-time profile and backend
|
||||||
|
validation.
|
||||||
|
|
||||||
|
## Priority 3: Semantic Execution-Target Fingerprints
|
||||||
|
|
||||||
|
**Disposition:** Deferred until prompt-independent profile inspection defines
|
||||||
|
the resolved target whose configuration identity would be fingerprinted.
|
||||||
|
|
||||||
|
### Downstream need
|
||||||
|
|
||||||
|
Notarius checkpoints model-backed pipeline stages. A checkpoint must not be
|
||||||
|
reused when generation-affecting PromptKit configuration changes.
|
||||||
|
|
||||||
|
Notarius therefore needs a stable equality signal for the effective profile
|
||||||
|
and backend target used by a pipeline.
|
||||||
|
|
||||||
|
### Current integration
|
||||||
|
|
||||||
|
Notarius currently constructs this identity itself from:
|
||||||
|
|
||||||
|
- a manually maintained marker for the PromptKit release and built-in profile
|
||||||
|
catalog;
|
||||||
|
- raw hashes of configured profile files; and
|
||||||
|
- a separate hash of the configured conventional local-backend endpoint.
|
||||||
|
|
||||||
|
This is safe but conservative and coupled to PromptKit details. Raw file
|
||||||
|
hashing also invalidates checkpoints for semantically irrelevant YAML changes,
|
||||||
|
such as comments or formatting.
|
||||||
|
|
||||||
|
### Requested capability
|
||||||
|
|
||||||
|
Expose an opaque semantic digest for a resolved profile and its effective
|
||||||
|
generation target. It could be returned by the proposed profile-resolution
|
||||||
|
API:
|
||||||
|
|
||||||
|
```go
|
||||||
|
type ResolvedProfile struct {
|
||||||
|
ProfileID string
|
||||||
|
BackendID string
|
||||||
|
EffectiveTarget ExecutionTarget
|
||||||
|
ExecutionDigest string
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Alternatively, PromptKit could expose a dedicated method such as
|
||||||
|
`ProfileExecutionDigest(profileID)`.
|
||||||
|
|
||||||
|
### Desired equality semantics
|
||||||
|
|
||||||
|
The digest should change when generation-affecting state changes, including:
|
||||||
|
|
||||||
|
- resolved model and endpoint;
|
||||||
|
- backend routing identity;
|
||||||
|
- backend request defaults and extra parameters;
|
||||||
|
- profile generation parameters; and
|
||||||
|
- the semantic identity of any selected built-in profile.
|
||||||
|
|
||||||
|
The digest should not incorporate:
|
||||||
|
|
||||||
|
- credential values;
|
||||||
|
- concurrency or queue capacity;
|
||||||
|
- filesystem source paths;
|
||||||
|
- YAML comments or formatting; or
|
||||||
|
- other settings that affect scheduling or source representation without
|
||||||
|
changing the generation target.
|
||||||
|
|
||||||
|
The credential environment-variable name may need to participate if changing
|
||||||
|
it can select a materially different provider account or target. PromptKit
|
||||||
|
should define this deliberately while continuing to exclude the resolved
|
||||||
|
secret value.
|
||||||
|
|
||||||
|
### Design considerations
|
||||||
|
|
||||||
|
- Treat the digest as an opaque equality value rather than a public encoding
|
||||||
|
of internal structures.
|
||||||
|
- Document which categories of change affect equality.
|
||||||
|
- Include a versioned semantic marker internally so PromptKit can deliberately
|
||||||
|
invalidate old digests when its resolution semantics change.
|
||||||
|
- Prefer a per-profile digest over a digest of every profile known to an
|
||||||
|
engine. Notarius generally knows which profiles a resolved pipeline uses.
|
||||||
|
- Do not require consumers to know PromptKit's built-in catalog version.
|
||||||
|
|
||||||
|
### Value to Notarius
|
||||||
|
|
||||||
|
This would let Notarius remove its PromptKit release marker and raw
|
||||||
|
profile-source fingerprinting, reduce unnecessary checkpoint invalidation, and
|
||||||
|
delegate execution-target equality to the component that owns target
|
||||||
|
resolution.
|
||||||
|
|
||||||
|
## Priority 4: Structured Capacity Errors
|
||||||
|
|
||||||
|
**Disposition:** Accepted into the
|
||||||
|
[future catalog](future.md#structured-capacity-errors).
|
||||||
|
|
||||||
|
### Downstream need
|
||||||
|
|
||||||
|
Notarius translates PromptKit backend-capacity rejection into a
|
||||||
|
provider-neutral application error. When multiple backends are active,
|
||||||
|
operators would benefit from knowing which backend rejected admission without
|
||||||
|
parsing an error string or exposing endpoint details.
|
||||||
|
|
||||||
|
### Current integration
|
||||||
|
|
||||||
|
PromptKit provides the useful `ErrCapacityExceeded` sentinel. Notarius can
|
||||||
|
classify the failure reliably, but it retains only a sanitized diagnostic
|
||||||
|
string as additional context.
|
||||||
|
|
||||||
|
### Requested capability
|
||||||
|
|
||||||
|
Add a typed error that continues to match `ErrCapacityExceeded`:
|
||||||
|
|
||||||
|
```go
|
||||||
|
type CapacityError struct {
|
||||||
|
BackendID string
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *CapacityError) Is(target error) bool {
|
||||||
|
return target == ErrCapacityExceeded
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The exact implementation may use `Unwrap` or another idiomatic mechanism. The
|
||||||
|
important properties are compatibility with `errors.Is` and discoverability
|
||||||
|
through `errors.As`.
|
||||||
|
|
||||||
|
### Design considerations
|
||||||
|
|
||||||
|
- Include the stable backend ID.
|
||||||
|
- Do not expose the backend endpoint, credential environment, credential
|
||||||
|
value, request content, or other sensitive configuration.
|
||||||
|
- Consider including the configured concurrency and queue limits if they are
|
||||||
|
useful and safe, but backend identity alone provides most of the downstream
|
||||||
|
value.
|
||||||
|
- Add a retry delay only if PromptKit can provide a meaningful value. A full
|
||||||
|
queue does not necessarily imply a reliable `Retry-After` duration.
|
||||||
|
- Keep retry and backoff policy with the consuming application. PromptKit
|
||||||
|
should classify the admission failure rather than silently retry it.
|
||||||
|
|
||||||
|
### Value to Notarius
|
||||||
|
|
||||||
|
This would improve operational diagnostics and future metrics while preserving
|
||||||
|
the provider-neutral error boundary used by Notarius.
|
||||||
|
|
||||||
|
## Capabilities PromptKit Already Provides Well
|
||||||
|
|
||||||
|
The current PromptKit boundary is sufficient for Notarius's implemented
|
||||||
|
behavior. In particular, PromptKit already provides:
|
||||||
|
|
||||||
|
- filesystem, `fs.FS`, and programmatic prompt, profile, and schema sources;
|
||||||
|
- offline preparation without model execution;
|
||||||
|
- structured output and content validation;
|
||||||
|
- direct session propagation;
|
||||||
|
- tri-state per-run reasoning overrides;
|
||||||
|
- selected profile, backend, model, endpoint, effective parameters, hashes, and
|
||||||
|
token-usage provenance;
|
||||||
|
- endpoint-only profiles;
|
||||||
|
- the conventional `local` backend helper;
|
||||||
|
- arbitrary engine-scoped `Backend` registrations;
|
||||||
|
- backend authentication environment names, extra parameters, concurrency
|
||||||
|
limits, and queue-capacity policies;
|
||||||
|
- provider-client and artifact-reader extension interfaces;
|
||||||
|
- context cancellation; and
|
||||||
|
- useful public error sentinels, including profile absence and capacity
|
||||||
|
exhaustion.
|
||||||
|
|
||||||
|
The wishlist does not imply that Notarius needs PromptKit to broaden its core
|
||||||
|
responsibilities. It primarily asks for more direct access to information and
|
||||||
|
operations that PromptKit already computes internally.
|
||||||
|
|
||||||
|
## Responsibilities That Should Remain In Notarius
|
||||||
|
|
||||||
|
The following concerns belong to the downstream application and should not
|
||||||
|
move into PromptKit for the sake of Notarius:
|
||||||
|
|
||||||
|
- pipeline staging, dependencies, and generated references;
|
||||||
|
- application-wide scheduling across providers and backends;
|
||||||
|
- module and validation retry policy;
|
||||||
|
- checkpoints, resume, and recomputation;
|
||||||
|
- durable run artifacts and manifests;
|
||||||
|
- D&D prompts, schemas, extractors, validators, and normalizers;
|
||||||
|
- Notarius configuration-file parsing and precedence;
|
||||||
|
- domain-specific prompt-cache prefix policy; and
|
||||||
|
- application-specific redaction, retention, and debug-bundle policy.
|
||||||
|
|
||||||
|
PromptKit's complete `Backend` API already supports custom IDs, multiple local
|
||||||
|
endpoints, authentication, extra parameters, and explicit queue policies.
|
||||||
|
Whether Notarius exposes those capabilities in its own configuration is an
|
||||||
|
application-policy decision, not an upstream PromptKit gap.
|
||||||
|
|
||||||
|
## Suggested Upstream Sequence
|
||||||
|
|
||||||
|
If the PromptKit team chooses to pursue these ideas, the most useful order for
|
||||||
|
Notarius would be:
|
||||||
|
|
||||||
|
1. Add atomic execution that returns prepared details and the completed result.
|
||||||
|
2. Add prompt-independent profile inspection.
|
||||||
|
3. Add a semantic execution-target digest, preferably as part of profile
|
||||||
|
inspection.
|
||||||
|
4. Add a typed capacity error carrying backend identity.
|
||||||
|
|
||||||
|
The first two address concrete workarounds in current Notarius code. The third
|
||||||
|
would improve checkpoint correctness and reduce coupling. The fourth is
|
||||||
|
operational polish.
|
||||||
280
docs/roadmap/prepared-execution.md
Normal file
280
docs/roadmap/prepared-execution.md
Normal file
@@ -0,0 +1,280 @@
|
|||||||
|
# Executable Preparation Handles
|
||||||
|
|
||||||
|
**Status:** Accepted.
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
Allow a consumer to prepare one exact Promptkit execution, inspect and retain
|
||||||
|
its credential-redacted public preparation details, and later execute that
|
||||||
|
already-prepared work without resolving or rendering the request again.
|
||||||
|
|
||||||
|
This provides a supported preflight-before-generation boundary for
|
||||||
|
[Weatherreporter](weatherreporter-promptkit-wishlist.md#priority-1-executable-preparation-handles)
|
||||||
|
and removes the duplicate `Prepare`-then-`Run` workaround described by
|
||||||
|
[Notarius](notarius-promptkit-wishlist.md#priority-1-atomic-execution-with-prepared-details).
|
||||||
|
|
||||||
|
## Motivation
|
||||||
|
|
||||||
|
`Engine.Prepare` currently returns the provenance and rendered details that
|
||||||
|
consumers need for debugging, persistence, and preflight checks. `Engine.Run`
|
||||||
|
then performs its own preparation before generation. A consumer that needs
|
||||||
|
both values must therefore prepare the same logical request twice.
|
||||||
|
|
||||||
|
That workaround duplicates source loading, input hashing, schema work, and
|
||||||
|
template rendering. It also permits filesystem-backed prompts, profiles,
|
||||||
|
schemas, or inputs to change between the public preparation and the
|
||||||
|
preparation that actually produces the result.
|
||||||
|
|
||||||
|
Promptkit already owns the complete preparation and execution pipeline. An
|
||||||
|
opt-in prepared-execution handle should expose the missing boundary without
|
||||||
|
putting rendered content into every `RunResult` or moving persistence policy
|
||||||
|
into the library.
|
||||||
|
|
||||||
|
## Consumer Workflow
|
||||||
|
|
||||||
|
The target public workflow is:
|
||||||
|
|
||||||
|
```go
|
||||||
|
prepared, err := engine.PrepareExecution(ctx, request)
|
||||||
|
if err != nil {
|
||||||
|
// Handle preparation failure.
|
||||||
|
return
|
||||||
|
}
|
||||||
|
defer prepared.Discard()
|
||||||
|
|
||||||
|
details := prepared.Details()
|
||||||
|
// Persist or inspect a consumer-selected safe subset of details.
|
||||||
|
|
||||||
|
result, err := engine.RunPrepared(ctx, prepared)
|
||||||
|
```
|
||||||
|
|
||||||
|
The target public surface is:
|
||||||
|
|
||||||
|
```go
|
||||||
|
type PreparedExecution struct {
|
||||||
|
// Opaque Promptkit-owned state.
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) PrepareExecution(
|
||||||
|
ctx context.Context,
|
||||||
|
req RunRequest,
|
||||||
|
) (*PreparedExecution, error)
|
||||||
|
|
||||||
|
func (p *PreparedExecution) Details() PreparedRun
|
||||||
|
|
||||||
|
func (p *PreparedExecution) Discard()
|
||||||
|
|
||||||
|
func (e *Engine) RunPrepared(
|
||||||
|
ctx context.Context,
|
||||||
|
prepared *PreparedExecution,
|
||||||
|
) (*RunResult, error)
|
||||||
|
```
|
||||||
|
|
||||||
|
The declarations and GoDoc will own the exact implemented contract. The
|
||||||
|
important public shape is an opaque handle, caller-owned `PreparedRun`
|
||||||
|
details, an explicit discard operation, and execution through the engine that
|
||||||
|
created the handle.
|
||||||
|
|
||||||
|
## Prepared Snapshot
|
||||||
|
|
||||||
|
`PrepareExecution` performs complete preparation without model generation or
|
||||||
|
backend-capacity admission. It applies the same request validation, source
|
||||||
|
precedence, backend and profile resolution, credential requirement checks,
|
||||||
|
output-contract resolution, artifact loading, hashing, schema loading,
|
||||||
|
session resolution, and rendering behavior as `Prepare`.
|
||||||
|
|
||||||
|
A successful handle freezes all source-derived state required for later
|
||||||
|
execution, including:
|
||||||
|
|
||||||
|
- the selected prompt definition, profile, and backend identity;
|
||||||
|
- the complete effective execution target and request-field presence;
|
||||||
|
- rendered messages and effective session ID;
|
||||||
|
- prompt, rendered-prompt, and input hashes;
|
||||||
|
- the effective output contract and provider-facing structured-output
|
||||||
|
constraint; and
|
||||||
|
- private validation state sufficient to validate generated output without
|
||||||
|
reopening schema files or `fs.FS` resources, including required schema
|
||||||
|
references.
|
||||||
|
|
||||||
|
After `PrepareExecution` succeeds, changes to prompt, profile, schema, input,
|
||||||
|
or request-owned data cannot change what `RunPrepared` sends to the model or
|
||||||
|
how it validates the generated output.
|
||||||
|
|
||||||
|
The handle retains an internal snapshot independent from values returned by
|
||||||
|
`Details`. Mutating a returned `PreparedRun`, its maps, slices, messages, or
|
||||||
|
schema values does not affect later execution. Each `Details` call returns a
|
||||||
|
fresh caller-owned copy under the existing `PreparedRun` ownership and stable
|
||||||
|
JSON rules.
|
||||||
|
|
||||||
|
## Handle Lifecycle
|
||||||
|
|
||||||
|
A `PreparedExecution` is:
|
||||||
|
|
||||||
|
- created only by a successful `PrepareExecution` call;
|
||||||
|
- bound to the exact `Engine` that created it;
|
||||||
|
- valid for one `RunPrepared` invocation;
|
||||||
|
- safe for repeated `Details` calls;
|
||||||
|
- intentionally opaque and without a supported JSON representation; and
|
||||||
|
- in-process state rather than a durable or restartable job.
|
||||||
|
|
||||||
|
`RunPrepared` atomically claims a valid handle before beginning the execution
|
||||||
|
attempt. A second or concurrent invocation fails without starting another
|
||||||
|
execution, including when the first invocation ended in cancellation, capacity
|
||||||
|
rejection, generation failure, or operational validation failure. Copies of
|
||||||
|
the public handle share the same one-attempt state and cannot bypass this rule.
|
||||||
|
|
||||||
|
A nil, zero-value, foreign-engine, discarded, already-claimed, or already-used
|
||||||
|
handle is invalid. `RunPrepared` reports these lifecycle errors through the
|
||||||
|
ordinary public invalid-request category. A failed foreign-engine invocation
|
||||||
|
does not consume a handle that remains valid for its owning engine.
|
||||||
|
|
||||||
|
`Discard` idempotently makes an unclaimed handle unavailable for execution and
|
||||||
|
drops Promptkit's references to secret-bearing or execution-only state.
|
||||||
|
`RunPrepared` performs the same cleanup automatically after claiming a handle.
|
||||||
|
Credential-redacted public preparation details remain available after discard,
|
||||||
|
success, or failure so consumers can retain diagnostics. Promptkit does not
|
||||||
|
promise secure erasure of Go string memory.
|
||||||
|
|
||||||
|
Lifecycle transitions are concurrency-safe. When `RunPrepared` and `Discard`
|
||||||
|
race, exactly one claims the ready handle. `Discard` is not an execution
|
||||||
|
cancellation mechanism and does not interrupt an attempt that has already
|
||||||
|
claimed the handle; consumers cancel that attempt through its context.
|
||||||
|
|
||||||
|
## Credentials And Sensitive Data
|
||||||
|
|
||||||
|
`Details` has the same security contract as `PreparedRun`: it can contain
|
||||||
|
rendered messages, schemas, identifiers, and hashes, but never a resolved API
|
||||||
|
key value. Consumers remain responsible for selecting, redacting, storing, and
|
||||||
|
retaining any persisted preparation material.
|
||||||
|
|
||||||
|
A direct `RunRequest.APIKey` is retained only in opaque execution state until
|
||||||
|
the handle is run or discarded. It is never added to details, hashes, JSON,
|
||||||
|
`String`, or `GoString` output.
|
||||||
|
|
||||||
|
An environment-variable name is frozen as part of the effective target, but
|
||||||
|
its credential value is not captured for the lifetime of the handle.
|
||||||
|
`PrepareExecution` applies the existing preparation-time availability check.
|
||||||
|
`RunPrepared` rechecks availability before admission, and the selected model
|
||||||
|
client uses the environment value visible during execution. This preserves
|
||||||
|
current secret ownership and avoids retaining an environment credential while
|
||||||
|
a consumer persists preflight material.
|
||||||
|
|
||||||
|
The opaque handle must not expose retained request data or credentials through
|
||||||
|
default formatting, JSON, or error messages.
|
||||||
|
|
||||||
|
## Admission, Cancellation, And Execution
|
||||||
|
|
||||||
|
`PrepareExecution` never reserves backend admission or an active-generation
|
||||||
|
permit. Its context governs preparation only; cancellation after it returns
|
||||||
|
does not invalidate the handle.
|
||||||
|
|
||||||
|
`RunPrepared` uses its own context for credential revalidation, backend
|
||||||
|
admission, active-generation waiting, model generation, output validation, and
|
||||||
|
any internal repair. Admission occurs when `RunPrepared` begins so a consumer
|
||||||
|
cannot occupy bounded capacity while inspecting or persisting preparation
|
||||||
|
details.
|
||||||
|
|
||||||
|
For a limited backend, the admission lease covers the complete prepared
|
||||||
|
execution attempt after admission: generation, validation, internal repair,
|
||||||
|
and every success or failure exit. Actual generation continues to use the
|
||||||
|
backend's FIFO active-generation permit. Existing capacity error identity,
|
||||||
|
cancellation behavior, and release guarantees remain in force.
|
||||||
|
|
||||||
|
Because a prepared handle is one-attempt, cancellation or capacity rejection
|
||||||
|
does not make it reusable. Retry and backoff policy remains with the consumer,
|
||||||
|
which may create a new prepared handle when another attempt is appropriate.
|
||||||
|
|
||||||
|
## Results And Failures
|
||||||
|
|
||||||
|
On success, `RunPrepared` returns the existing caller-owned `RunResult`. Its
|
||||||
|
source-derived provenance must match `Details`, including prompt identity and
|
||||||
|
hash, rendered-prompt hash, session ID, selected profile and backend,
|
||||||
|
effective target, and input hashes.
|
||||||
|
|
||||||
|
`RunResult.StartTime`, `EndTime`, and `Duration` describe the
|
||||||
|
`RunPrepared` execution attempt. They exclude preparation time and any delay
|
||||||
|
while the consumer retained the handle. Preparation timing remains in
|
||||||
|
`PreparedRun`.
|
||||||
|
|
||||||
|
Preparation failure returns no handle. After successful preparation,
|
||||||
|
`RunPrepared` retains the existing rule that an operational failure returns no
|
||||||
|
partial `RunResult`; the consumer already has independent preparation details.
|
||||||
|
A completed content-validation failure remains a successful result with
|
||||||
|
`ValidationFailed`.
|
||||||
|
|
||||||
|
`PrepareExecution` preserves the public error categories of `Prepare`.
|
||||||
|
`RunPrepared` preserves applicable invalid-request, credential, capacity,
|
||||||
|
generation, validation, collaborator, and cancellation identities without
|
||||||
|
reintroducing source-loading or rendering failures from frozen state.
|
||||||
|
|
||||||
|
## Compatibility And Existing Workflows
|
||||||
|
|
||||||
|
This feature is additive:
|
||||||
|
|
||||||
|
- `Prepare` remains the simple preparation-only operation;
|
||||||
|
- `Run` remains the simple prepare-and-execute operation with its current
|
||||||
|
early-admission and error-ordering behavior;
|
||||||
|
- `PreparedRun` and `RunResult` retain their existing stable JSON
|
||||||
|
representations;
|
||||||
|
- model-client and artifact-reader extension interfaces remain unchanged; and
|
||||||
|
- backend routing, concurrency limits, queue capacities, and provider wire
|
||||||
|
behavior remain unchanged.
|
||||||
|
|
||||||
|
The new workflow may share internal machinery with `Prepare` and `Run`, but it
|
||||||
|
must not change their observable behavior merely to simplify implementation.
|
||||||
|
|
||||||
|
## Documentation
|
||||||
|
|
||||||
|
The completed documentation set has these ownership boundaries:
|
||||||
|
|
||||||
|
- exported declarations and GoDoc own the exact handle, method, lifecycle,
|
||||||
|
ownership, concurrency, credential, error, and cancellation contracts;
|
||||||
|
- the promptkit consumer guide explains when to use `Prepare`, `Run`, or the
|
||||||
|
two-phase prepared-execution workflow; and
|
||||||
|
- internal runner, source-validation, capacity, and model-client documentation
|
||||||
|
describe the implemented collaborator boundaries without duplicating public
|
||||||
|
contracts.
|
||||||
|
|
||||||
|
No release document is part of the feature implementation itself. Release
|
||||||
|
guidance is prepared only when the resulting public API is selected for
|
||||||
|
publication.
|
||||||
|
|
||||||
|
## Non-Goals
|
||||||
|
|
||||||
|
This work does not include:
|
||||||
|
|
||||||
|
- serializable, durable, resumable, or cross-process execution handles;
|
||||||
|
- reuse for multiple consumer-initiated executions;
|
||||||
|
- concurrent execution of one handle;
|
||||||
|
- capacity reservation during preparation;
|
||||||
|
- a background task queue, priorities, worker lifecycle, or job status;
|
||||||
|
- retry, backoff, or provider failover policy;
|
||||||
|
- a separate `RunDetailed` convenience method;
|
||||||
|
- adding preparation details or rendered messages to every `RunResult`;
|
||||||
|
- prompt or profile inspection APIs;
|
||||||
|
- structured capacity or generation errors;
|
||||||
|
- freezing environment-variable credential values for the handle lifetime;
|
||||||
|
- snapshotting provider state or mutable behavior inside an injected
|
||||||
|
`LLMClient`;
|
||||||
|
- allowing target, validation, session, variable, or input overrides after
|
||||||
|
preparation; or
|
||||||
|
- changing existing `Prepare`, `Run`, file-format, provider-request, or stable
|
||||||
|
JSON contracts.
|
||||||
|
|
||||||
|
## Target End State
|
||||||
|
|
||||||
|
After this work:
|
||||||
|
|
||||||
|
- consumers can perform and persist preflight before starting provider work;
|
||||||
|
- one prepared handle executes exactly the source-derived prompt, target,
|
||||||
|
schema, inputs, session, and messages described by its public details;
|
||||||
|
- execution never reloads or rerenders consumer sources;
|
||||||
|
- direct credentials remain confined to opaque, explicitly discardable state;
|
||||||
|
- environment credentials are not retained across the preflight boundary;
|
||||||
|
- backend capacity is reserved only when execution begins;
|
||||||
|
- one handle can start at most one execution attempt, including any internal
|
||||||
|
repair calls owned by that attempt;
|
||||||
|
- preparation details remain available after execution success or failure;
|
||||||
|
- existing simple `Prepare` and `Run` consumers remain unaffected; and
|
||||||
|
- Promptkit continues to own reusable execution mechanics without taking on
|
||||||
|
downstream persistence, redaction, retry, or job-management policy.
|
||||||
468
docs/roadmap/weatherreporter-promptkit-wishlist.md
Normal file
468
docs/roadmap/weatherreporter-promptkit-wishlist.md
Normal file
@@ -0,0 +1,468 @@
|
|||||||
|
# Weatherreporter PromptKit Wishlist
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
This document records features and interface changes that would be useful
|
||||||
|
additions to PromptKit from the perspective of the maintainers of
|
||||||
|
Weatherreporter, a downstream application planning to replace its Scriptorium
|
||||||
|
CLI integration with PromptKit.
|
||||||
|
|
||||||
|
PromptKit v0.3.0 provides the capabilities Weatherreporter needs for the
|
||||||
|
migration. None of the ideas below is a hard adoption requirement. They are
|
||||||
|
opportunities to avoid duplicate preparation, validate configuration earlier,
|
||||||
|
improve durable failure diagnostics, and make the integration more direct.
|
||||||
|
|
||||||
|
The examples are API sketches intended to communicate the desired capability,
|
||||||
|
not prescriptive names or finalized Go contracts. The related
|
||||||
|
[Notarius PromptKit wishlist](notarius-promptkit-wishlist.md) proposes several
|
||||||
|
overlapping features from another downstream consumer's perspective.
|
||||||
|
|
||||||
|
## Priority 1: Executable Preparation Handles
|
||||||
|
|
||||||
|
**Disposition:** Accepted into the
|
||||||
|
[executable preparation handles](prepared-execution.md) feature roadmap.
|
||||||
|
|
||||||
|
### Downstream need
|
||||||
|
|
||||||
|
Weatherreporter treats prompt preparation as a durable preflight boundary. It
|
||||||
|
needs to:
|
||||||
|
|
||||||
|
1. prepare the exact request that will be executed;
|
||||||
|
2. persist a safe preparation record before starting the provider call; and
|
||||||
|
3. execute without reloading or rerendering prompt, profile, schema, or input
|
||||||
|
sources.
|
||||||
|
|
||||||
|
Persisting preflight before generation leaves useful evidence when a provider
|
||||||
|
call fails or the process is interrupted during generation.
|
||||||
|
|
||||||
|
### Current integration option
|
||||||
|
|
||||||
|
With PromptKit v0.3.0, Weatherreporter can call `Engine.Prepare`, save selected
|
||||||
|
fields from the returned `PreparedRun`, and then call `Engine.Run` with the
|
||||||
|
same request. Because `Run` performs preparation internally, the work is
|
||||||
|
repeated.
|
||||||
|
|
||||||
|
Weatherreporter plans to use embedded prompt and schema files plus immutable
|
||||||
|
inline input bytes, which removes most of the consistency risk. An external
|
||||||
|
profile file or directory can still change between the two calls, and the
|
||||||
|
second preparation remains unnecessary work.
|
||||||
|
|
||||||
|
The atomic `RunDetailed` operation proposed by the
|
||||||
|
[Notarius wishlist](notarius-promptkit-wishlist.md#priority-1-atomic-execution-with-prepared-details)
|
||||||
|
would guarantee that returned preparation details describe the completed
|
||||||
|
execution. However, returning those details only after generation would not
|
||||||
|
preserve Weatherreporter's preflight-before-generation persistence boundary.
|
||||||
|
|
||||||
|
### Requested capability
|
||||||
|
|
||||||
|
Add an opt-in two-phase API that returns a prepared execution handle:
|
||||||
|
|
||||||
|
```go
|
||||||
|
prepared, err := engine.PrepareExecution(ctx, request)
|
||||||
|
if err != nil {
|
||||||
|
// Handle preparation failure.
|
||||||
|
}
|
||||||
|
|
||||||
|
details := prepared.Details()
|
||||||
|
// Persist a consumer-selected safe preparation record.
|
||||||
|
|
||||||
|
result, err := engine.RunPrepared(ctx, prepared)
|
||||||
|
```
|
||||||
|
|
||||||
|
The exact names and shapes are flexible. The important contract is that
|
||||||
|
`RunPrepared` executes the already prepared prompt and does not reload or
|
||||||
|
rerender its prompt, profile, schema, or input sources.
|
||||||
|
|
||||||
|
`Details` should return the same caller-owned public preparation information
|
||||||
|
currently represented by `PreparedRun`. The execution handle may retain opaque
|
||||||
|
engine-owned state needed to invoke the model and validate the response.
|
||||||
|
|
||||||
|
### Design considerations
|
||||||
|
|
||||||
|
- Keep `Prepare` and `Run` available for consumers that do not need a
|
||||||
|
two-phase execution boundary.
|
||||||
|
- Bind a prepared handle to the engine that constructed it.
|
||||||
|
- Define whether a handle is one-shot, reusable, or safe for concurrent use.
|
||||||
|
A one-shot contract may be the safest initial design.
|
||||||
|
- Do not give the opaque handle a stable JSON representation.
|
||||||
|
- Do not expose or serialize resolved credential values through `Details`.
|
||||||
|
- Define how a direct request API key is retained and released when an opaque
|
||||||
|
handle must carry it until execution.
|
||||||
|
- Preserve caller-owned copies for all public details.
|
||||||
|
- Make context cancellation and backend admission timing explicit.
|
||||||
|
- Document whether profile credential environment values are resolved during
|
||||||
|
preparation or execution.
|
||||||
|
- Ensure an execution error does not invalidate the public details already
|
||||||
|
returned to the consumer.
|
||||||
|
- Consider whether an atomic `RunDetailed` can share the same internal
|
||||||
|
prepared-execution implementation.
|
||||||
|
|
||||||
|
### Value to Weatherreporter
|
||||||
|
|
||||||
|
This is the highest-value upstream addition. It would preserve
|
||||||
|
Weatherreporter's durable preflight behavior, remove duplicate work, eliminate
|
||||||
|
the remaining source-consistency window, and ensure that persisted provenance
|
||||||
|
describes the actual execution.
|
||||||
|
|
||||||
|
## Priority 2: Prompt-Definition Inspection
|
||||||
|
|
||||||
|
**Disposition:** Accepted into the
|
||||||
|
[future catalog](future.md#prompt-definition-inspection).
|
||||||
|
|
||||||
|
### Downstream need
|
||||||
|
|
||||||
|
Weatherreporter has a fixed registry of seven report definitions. Each report
|
||||||
|
selects a prompt ID and one of two output workflows:
|
||||||
|
|
||||||
|
- direct Markdown; or
|
||||||
|
- structured generated text followed by application-owned domain validation
|
||||||
|
and Markdown template rendering.
|
||||||
|
|
||||||
|
Weatherreporter will embed the PromptKit prompt definitions and private
|
||||||
|
response schemas that implement those reports. It needs to validate that the
|
||||||
|
report registry and embedded prompt corpus agree before weather collection or
|
||||||
|
provider execution.
|
||||||
|
|
||||||
|
### Current integration option
|
||||||
|
|
||||||
|
Weatherreporter can maintain synthetic data-package fixtures and call
|
||||||
|
`Engine.Prepare` for every report prompt during tests. Runtime validation can
|
||||||
|
also occur through the ordinary per-report preparation stage.
|
||||||
|
|
||||||
|
This works, but it requires complete placeholder inputs and profile resolution
|
||||||
|
when the application primarily wants to inspect prompt identity and declared
|
||||||
|
contracts.
|
||||||
|
|
||||||
|
### Requested capability
|
||||||
|
|
||||||
|
Add exact prompt-definition lookup without rendering or generation:
|
||||||
|
|
||||||
|
```go
|
||||||
|
type PromptInfo struct {
|
||||||
|
PromptID string
|
||||||
|
PromptVersion string
|
||||||
|
PromptHash string
|
||||||
|
DefaultProfileID string
|
||||||
|
Inputs []InputDefinition
|
||||||
|
OutputContract OutputContract
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) ResolvePrompt(
|
||||||
|
ctx context.Context,
|
||||||
|
promptID string,
|
||||||
|
promptVersion string,
|
||||||
|
) (PromptInfo, error)
|
||||||
|
```
|
||||||
|
|
||||||
|
The exact returned shape may differ. Weatherreporter needs enough information
|
||||||
|
to verify prompt existence, version selection, declared inputs, default
|
||||||
|
profile identity, output format, validation mode, and schema selection without
|
||||||
|
supplying synthetic prompt input.
|
||||||
|
|
||||||
|
### Design considerations
|
||||||
|
|
||||||
|
- Use ordinary PromptKit prompt-source precedence and exact ID/version
|
||||||
|
selection.
|
||||||
|
- Fully load and structurally validate the selected prompt definition.
|
||||||
|
- Validate referenced prompt content files without rendering their templates.
|
||||||
|
- Resolve and validate the selected output contract and schema reference where
|
||||||
|
practical.
|
||||||
|
- Return an opaque prompt-definition equality value rather than raw source
|
||||||
|
bytes.
|
||||||
|
- Do not return rendered messages, schema bodies, profile credentials, or
|
||||||
|
another source of sensitive content.
|
||||||
|
- Preserve typed or sentinel errors for missing and invalid prompts.
|
||||||
|
- Return caller-owned values.
|
||||||
|
- Enumeration of all known prompts is not required for Weatherreporter; exact
|
||||||
|
lookup is sufficient.
|
||||||
|
|
||||||
|
### Value to Weatherreporter
|
||||||
|
|
||||||
|
This would let Weatherreporter directly verify that every report prompt
|
||||||
|
exists, requires the curated `data_package` input, and declares the expected
|
||||||
|
Markdown or JSON Schema output contract. It would reduce synthetic test setup
|
||||||
|
and move failures ahead of weather collection.
|
||||||
|
|
||||||
|
## Priority 3: Prompt-Independent Profile Inspection
|
||||||
|
|
||||||
|
**Disposition:** Accepted into the
|
||||||
|
[future catalog](future.md#prompt-independent-profile-inspection).
|
||||||
|
|
||||||
|
### Downstream need
|
||||||
|
|
||||||
|
Weatherreporter will allow operators to select an external PromptKit profile
|
||||||
|
source and may allow an explicit profile override. It should reject a missing
|
||||||
|
profile, unknown backend, malformed execution target, or unsatisfied credential
|
||||||
|
requirement before collecting weather data or writing report artifacts.
|
||||||
|
|
||||||
|
### Current integration option
|
||||||
|
|
||||||
|
Weatherreporter can validate an explicit profile by preparing one embedded
|
||||||
|
prompt with fixture input. Prompts that use their own default profiles can be
|
||||||
|
validated during their normal preparation stage.
|
||||||
|
|
||||||
|
This couples configuration validation to one prompt and requires placeholder
|
||||||
|
input even when only profile and backend resolution are relevant.
|
||||||
|
|
||||||
|
### Requested capability
|
||||||
|
|
||||||
|
The prompt-independent `ResolveProfile` API proposed by the
|
||||||
|
[Notarius wishlist](notarius-promptkit-wishlist.md#priority-2-prompt-independent-profile-inspection)
|
||||||
|
would satisfy this need. It should resolve built-in, file-backed, and
|
||||||
|
programmatic profiles, validate backend membership, report credential
|
||||||
|
requirements without resolving credential values, and preserve typed error
|
||||||
|
classification.
|
||||||
|
|
||||||
|
### Additional Weatherreporter considerations
|
||||||
|
|
||||||
|
- An explicit application profile override should be inspectable without
|
||||||
|
selecting a report prompt.
|
||||||
|
- A prompt-definition inspection result may expose its default profile ID so
|
||||||
|
Weatherreporter can inspect that profile separately.
|
||||||
|
- Inspection should distinguish structural profile validity from current
|
||||||
|
credential availability so configuration validation can apply explicit
|
||||||
|
application policy.
|
||||||
|
- An optional execution-target override should be considered only if it
|
||||||
|
describes the same target that a later run will use.
|
||||||
|
|
||||||
|
### Value to Weatherreporter
|
||||||
|
|
||||||
|
This would improve fail-fast configuration validation and give operator-facing
|
||||||
|
errors direct profile and backend context. It is valuable but not required for
|
||||||
|
the initial migration.
|
||||||
|
|
||||||
|
## Priority 4: Eager Source Validation
|
||||||
|
|
||||||
|
**Disposition:** Deferred until prompt and profile inspection have been used
|
||||||
|
to determine whether a broader engine-wide validation operation is still
|
||||||
|
needed.
|
||||||
|
|
||||||
|
### Downstream need
|
||||||
|
|
||||||
|
PromptKit deliberately defers reading and validating filesystem and `fs.FS`
|
||||||
|
prompt, profile, and schema content until a request needs it. Weatherreporter
|
||||||
|
has a small fixed embedded prompt corpus and one optional external profile
|
||||||
|
source. It would benefit from an explicit offline validation operation for
|
||||||
|
tests, startup diagnostics, and configuration checks.
|
||||||
|
|
||||||
|
### Current integration option
|
||||||
|
|
||||||
|
Weatherreporter can prepare every report prompt with fixture inputs and inspect
|
||||||
|
any explicit profiles individually. That provides strong coverage but requires
|
||||||
|
consumer-maintained traversal and synthetic material.
|
||||||
|
|
||||||
|
### Requested capability
|
||||||
|
|
||||||
|
Consider an opt-in source-validation operation:
|
||||||
|
|
||||||
|
```go
|
||||||
|
type SourceValidationOptions struct {
|
||||||
|
RequireCredentials bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func (e *Engine) ValidateSources(
|
||||||
|
ctx context.Context,
|
||||||
|
opts SourceValidationOptions,
|
||||||
|
) error
|
||||||
|
```
|
||||||
|
|
||||||
|
The operation should eagerly discover and structurally validate the configured
|
||||||
|
prompt, profile, and schema sources without model generation.
|
||||||
|
|
||||||
|
### Design considerations
|
||||||
|
|
||||||
|
- Keep deferred validation as the normal `NewEngine` behavior.
|
||||||
|
- Make eager validation an explicit consumer choice.
|
||||||
|
- Validate duplicate IDs and versions, strict YAML decoding, referenced content
|
||||||
|
files, profile/backend membership, schema syntax, and schema references.
|
||||||
|
- Distinguish structural credential declarations from current environment
|
||||||
|
availability.
|
||||||
|
- Do not read or expose credential values when credential availability is not
|
||||||
|
requested.
|
||||||
|
- Preserve source-specific public error identities and useful path context.
|
||||||
|
- Respect context cancellation during filesystem discovery and schema work.
|
||||||
|
- Consider whether exact prompt and profile inspection APIs already provide a
|
||||||
|
smaller sufficient surface before adding an engine-wide operation.
|
||||||
|
|
||||||
|
### Value to Weatherreporter
|
||||||
|
|
||||||
|
This would simplify offline corpus checks and catch malformed operator profile
|
||||||
|
sources before report work begins. It is helpful but lower priority than exact
|
||||||
|
prompt and profile inspection.
|
||||||
|
|
||||||
|
## Priority 5: Structured Generation Errors
|
||||||
|
|
||||||
|
**Disposition:** Deferred pending stronger downstream demand and a narrower
|
||||||
|
design that does not duplicate prepared provenance or impose HTTP-specific
|
||||||
|
fields on injected model clients.
|
||||||
|
|
||||||
|
### Downstream need
|
||||||
|
|
||||||
|
Weatherreporter preserves redacted, inspectable failure receipts for report
|
||||||
|
runs. When model generation fails operationally, it needs to classify the
|
||||||
|
failure and retain safe execution context without parsing error prose.
|
||||||
|
|
||||||
|
Prompt preparation already supplies selected profile, backend, and model
|
||||||
|
identity. Provider status classification would add useful operator context,
|
||||||
|
especially when the built-in OpenAI-compatible client receives a non-success
|
||||||
|
HTTP status.
|
||||||
|
|
||||||
|
### Current integration option
|
||||||
|
|
||||||
|
PromptKit exposes `ErrLLMGenerate` and preserves injected client errors through
|
||||||
|
`errors.Is`. Weatherreporter can reliably classify generation failure and use
|
||||||
|
its preparation record for profile, backend, and model provenance. Any further
|
||||||
|
diagnostic detail remains a redacted error string.
|
||||||
|
|
||||||
|
### Requested capability
|
||||||
|
|
||||||
|
Consider a typed generation error that continues to match `ErrLLMGenerate`:
|
||||||
|
|
||||||
|
```go
|
||||||
|
type GenerationError struct {
|
||||||
|
BackendID string
|
||||||
|
Model string
|
||||||
|
StatusCode int
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
The exact fields may differ. The useful contract is safe structured context
|
||||||
|
available through `errors.As`, while `errors.Is(err, ErrLLMGenerate)` remains
|
||||||
|
compatible.
|
||||||
|
|
||||||
|
### Design considerations
|
||||||
|
|
||||||
|
- Include only fields that PromptKit knows reliably and can expose safely.
|
||||||
|
- Treat an HTTP status as optional because injected model clients may not use
|
||||||
|
HTTP.
|
||||||
|
- Do not expose provider response bodies, endpoints, credential environment
|
||||||
|
names, credential values, request content, or generated content.
|
||||||
|
- Do not make a structured error a second source of prompt/profile provenance
|
||||||
|
already present in a prepared execution.
|
||||||
|
- Preserve injected client error identity.
|
||||||
|
- Keep retry and backoff policy with the consuming application.
|
||||||
|
|
||||||
|
### Value to Weatherreporter
|
||||||
|
|
||||||
|
This would improve durable failure receipts and troubleshooting, particularly
|
||||||
|
for built-in transport failures. It is not required if preparation details and
|
||||||
|
the existing sentinel remain available.
|
||||||
|
|
||||||
|
## Lower-Priority Shared Wishlist Items
|
||||||
|
|
||||||
|
### Structured Capacity Errors
|
||||||
|
|
||||||
|
**Disposition:** Accepted into the
|
||||||
|
[future catalog](future.md#structured-capacity-errors).
|
||||||
|
|
||||||
|
The typed capacity error proposed by the
|
||||||
|
[Notarius wishlist](notarius-promptkit-wishlist.md#priority-4-structured-capacity-errors)
|
||||||
|
would improve Weatherreporter diagnostics by exposing the stable backend ID
|
||||||
|
without parsing error text.
|
||||||
|
|
||||||
|
Weatherreporter currently generates batch reports sequentially and constructs
|
||||||
|
one engine per invocation, so engine-local capacity exhaustion is unlikely in
|
||||||
|
the initial design. The feature would become more valuable if report
|
||||||
|
generation later becomes concurrent or PromptKit engines become longer-lived.
|
||||||
|
It should not block adoption.
|
||||||
|
|
||||||
|
### Semantic Execution-Target Fingerprints
|
||||||
|
|
||||||
|
**Disposition:** Deferred until prompt-independent profile inspection defines
|
||||||
|
the resolved target whose configuration identity would be fingerprinted.
|
||||||
|
|
||||||
|
The semantic target digest proposed by the
|
||||||
|
[Notarius wishlist](notarius-promptkit-wishlist.md#priority-3-semantic-execution-target-fingerprints)
|
||||||
|
would provide a compact equality signal for audit metadata.
|
||||||
|
|
||||||
|
Weatherreporter does not currently reuse LLM-dependent checkpoints. Its Recent
|
||||||
|
Changes behavior compares deterministic module snapshots rather than generated
|
||||||
|
reports, so the digest has no immediate cache-correctness role. Existing
|
||||||
|
PromptKit result metadata is sufficient for the initial integration. A digest
|
||||||
|
would still be useful provenance and future-proofing, but it is not a
|
||||||
|
migration priority.
|
||||||
|
|
||||||
|
## Capabilities PromptKit Already Provides Well
|
||||||
|
|
||||||
|
PromptKit v0.3.0 already provides the essential Weatherreporter integration
|
||||||
|
surface:
|
||||||
|
|
||||||
|
- importable in-process engine construction;
|
||||||
|
- filesystem, `fs.FS`, and programmatic prompt, profile, and schema sources;
|
||||||
|
- offline preparation without model execution;
|
||||||
|
- versioned prompt selection;
|
||||||
|
- text, Markdown, JSON, and JSON Schema output contracts;
|
||||||
|
- single-pass output validation with raw output retained after completed
|
||||||
|
validation failure;
|
||||||
|
- inline input artifacts with provenance URIs and input hashes;
|
||||||
|
- selected profile, backend, model, effective target, prompt hashes, timing,
|
||||||
|
and token-usage provenance;
|
||||||
|
- endpoint-only profiles and engine-scoped backend registration;
|
||||||
|
- injected model-client and artifact-reader interfaces;
|
||||||
|
- caller cancellation, generation timeout, and transport timeout behavior; and
|
||||||
|
- public error sentinels for configuration, prompt, profile, artifact,
|
||||||
|
validation, capacity, and generation failures.
|
||||||
|
|
||||||
|
These capabilities are sufficient for Weatherreporter to adopt PromptKit
|
||||||
|
without waiting for new upstream work.
|
||||||
|
|
||||||
|
## Responsibilities That Should Remain In Weatherreporter
|
||||||
|
|
||||||
|
The following concerns belong to Weatherreporter and should not move into
|
||||||
|
PromptKit:
|
||||||
|
|
||||||
|
- report definitions, valid periods, batches, and output naming;
|
||||||
|
- application prompt content and private report response schemas;
|
||||||
|
- deterministic weather facts, modules, and Recent Changes;
|
||||||
|
- curated `data_package` construction and persistence;
|
||||||
|
- generated-text domain validation and Markdown template rendering;
|
||||||
|
- managed artifact paths, atomic writes, metadata, and inspection commands;
|
||||||
|
- preparation, execution, raw-output, and failure-receipt schemas;
|
||||||
|
- CLI configuration loading and precedence;
|
||||||
|
- debug enablement, redaction, placement, sensitivity, and retention;
|
||||||
|
- distributor notification;
|
||||||
|
- batch continuation and any future retry policy; and
|
||||||
|
- application-level compatibility and migration policy.
|
||||||
|
|
||||||
|
## Suggested Upstream Sequence
|
||||||
|
|
||||||
|
If the PromptKit team chooses to pursue these ideas, the most useful order for
|
||||||
|
Weatherreporter would be:
|
||||||
|
|
||||||
|
1. Add executable preparation handles, ideally sharing implementation with an
|
||||||
|
atomic detailed-run API.
|
||||||
|
2. Add prompt-definition inspection.
|
||||||
|
3. Add prompt-independent profile inspection.
|
||||||
|
4. Consider eager source validation after evaluating whether the two exact
|
||||||
|
inspection APIs are sufficient.
|
||||||
|
5. Add structured generation errors.
|
||||||
|
6. Add structured capacity errors and semantic execution-target fingerprints
|
||||||
|
as lower-priority operational improvements.
|
||||||
|
|
||||||
|
The first item removes the only material integration workaround. Prompt and
|
||||||
|
profile inspection improve fail-fast validation. The remaining items improve
|
||||||
|
ergonomics and diagnostics.
|
||||||
|
|
||||||
|
## Adoption Sequencing
|
||||||
|
|
||||||
|
Weatherreporter should not wait for the complete wishlist. PromptKit v0.3.0 is
|
||||||
|
already sufficient when Weatherreporter:
|
||||||
|
|
||||||
|
- embeds immutable prompt and schema assets;
|
||||||
|
- supplies immutable inline data-package bytes;
|
||||||
|
- constructs one engine per CLI invocation;
|
||||||
|
- calls `Prepare` and `Run` with the same request; and
|
||||||
|
- keeps PromptKit behind a weatherreporter-owned adapter contract.
|
||||||
|
|
||||||
|
If executable preparation handles are scheduled for a near-term PromptKit
|
||||||
|
release, Weatherreporter may defer only its final adapter implementation to
|
||||||
|
avoid implementing and then removing duplicate preparation. Prompt corpus
|
||||||
|
retrieval, application-contract design, configuration work, embedded assets,
|
||||||
|
state contracts, and offline fixtures can proceed independently.
|
||||||
|
|
||||||
|
If the feature is not scheduled, Weatherreporter can adopt v0.3.0 and keep the
|
||||||
|
duplicate `Prepare` and `Run` sequence inside its adapter. A later PromptKit
|
||||||
|
upgrade would remain localized behind that neutral boundary.
|
||||||
|
|
||||||
|
Prompt inspection, profile inspection, source validation, structured errors,
|
||||||
|
capacity details, and semantic fingerprints should not gate adoption.
|
||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"reflect"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
@@ -138,6 +139,49 @@ func TestUnknownProfileBackendHasProfileLoadIdentity(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLocalBackendConstructsAndRegistersConventionalBackend(t *testing.T) {
|
||||||
|
const (
|
||||||
|
localBackendID = "local"
|
||||||
|
limit = 2
|
||||||
|
)
|
||||||
|
if promptkit.BackendLocal != localBackendID {
|
||||||
|
t.Fatalf("BackendLocal=%q, want %q", promptkit.BackendLocal, localBackendID)
|
||||||
|
}
|
||||||
|
|
||||||
|
endpoint := "http://local.example/v1"
|
||||||
|
backend := promptkit.LocalBackend(endpoint, limit)
|
||||||
|
want := promptkit.Backend{
|
||||||
|
ID: localBackendID,
|
||||||
|
Endpoint: endpoint,
|
||||||
|
ConcurrencyLimit: limit,
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(backend, want) {
|
||||||
|
t.Fatalf("LocalBackend()=%+v, want %+v", backend, want)
|
||||||
|
}
|
||||||
|
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prompt", "local-profile", "message"), "."),
|
||||||
|
promptkit.WithBackend(backend),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "local-profile",
|
||||||
|
BackendID: localBackendID,
|
||||||
|
Model: "model",
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine with local backend: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare with local backend: %v", err)
|
||||||
|
}
|
||||||
|
if prepared.SelectedBackendID != localBackendID ||
|
||||||
|
prepared.EffectiveModelParams.Endpoint != endpoint {
|
||||||
|
t.Fatalf("unexpected local backend preparation: %+v", prepared)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestCustomBackendFlowsThroughProfilesOverridesAndInjectedClient(t *testing.T) {
|
func TestCustomBackendFlowsThroughProfilesOverridesAndInjectedClient(t *testing.T) {
|
||||||
t.Setenv("CUSTOM_LLM_KEY", "test-key")
|
t.Setenv("CUSTOM_LLM_KEY", "test-key")
|
||||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
|
||||||
|
|||||||
Reference in New Issue
Block a user