11 Commits

60 changed files with 3297 additions and 689 deletions

55
backends.go Normal file
View File

@@ -0,0 +1,55 @@
package promptkit
import (
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
// BackendOpenRouter is the reserved ID of Promptkit's built-in OpenRouter
// backend.
const BackendOpenRouter = backend.OpenRouterID
// Backend configures one engine-scoped OpenAI-compatible backend.
//
// Backend has no stable JSON representation. Use keyed literals so additions
// to this configuration value do not break source compatibility.
type Backend struct {
// ID is the stable, case-sensitive registry key. NewEngine trims it and
// requires a non-blank value. BackendOpenRouter is reserved.
ID string
// Endpoint is the OpenAI-compatible base endpoint. NewEngine trims it and
// requires an absolute HTTP or HTTPS URL with a host and without user
// information, a query string, or a fragment. Paths are allowed.
Endpoint string
// APIKeyEnv optionally names the environment variable containing the API
// key. NewEngine trims it and requires the portable form
// [A-Za-z_][A-Za-z0-9_]*. Store only the name, never a credential value.
APIKeyEnv string
// ExtraParams contains backend-wide request defaults. Values must be
// JSON-compatible, finite, acyclic, and keyed by non-empty strings. Keys
// must not be model, session_id, messages, temperature, max_tokens, top_p,
// service_tier, reasoning_effort, or response_format. An empty map supplies
// no defaults. NewEngine deeply copies the map.
ExtraParams map[string]any
}
// WithBackend adds one Backend registration to the constructed Engine.
//
// Registrations accumulate in option order. Every normalized ID must be unique
// across consumer registrations and built-ins; a duplicate or invalid
// definition makes NewEngine fail with ErrInvalidConfig. In particular,
// BackendOpenRouter cannot be replaced. The immutable registration is scoped
// to the resulting Engine and cannot be enumerated, replaced, removed, or
// mutated after construction. WithBackend does not install package-global
// state.
func WithBackend(backend Backend) Option {
return optionFunc(func(options *engineOptions) error {
options.backends = append(options.backends, domain.Backend{
ID: backend.ID,
Endpoint: backend.Endpoint,
APIKeyEnv: backend.APIKeyEnv,
ExtraParams: backend.ExtraParams,
})
return nil
})
}

View File

@@ -4,6 +4,7 @@ import (
"reflect"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
)
func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
@@ -15,6 +16,7 @@ func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
PromptID: req.PromptID,
PromptVersion: req.PromptVersion,
ProfileID: req.ProfileID,
SessionID: req.SessionID,
APIKey: req.APIKey,
Inputs: toDomainArtifactRefMap(req.Inputs),
Vars: copyStringMap(req.Vars),
@@ -32,6 +34,7 @@ func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
PromptVersion: prepared.PromptVersion,
PromptHash: prepared.PromptHash,
SelectedProfileID: prepared.SelectedProfileID,
SelectedBackendID: prepared.SelectedBackendID,
EffectiveModelParams: fromDomainExecutionTarget(prepared.EffectiveModelParams),
OutputContract: fromDomainOutputContract(prepared.OutputContract),
StructuredOutput: fromDomainStructuredOutputSpec(prepared.StructuredOutput),
@@ -57,8 +60,10 @@ func fromDomainRunResult(result *domain.RunResult) *RunResult {
PromptID: result.PromptID,
PromptVersion: result.PromptVersion,
PromptHash: result.PromptHash,
SessionID: result.SessionID,
RenderedPromptHash: result.RenderedPromptHash,
SelectedProfileID: result.SelectedProfileID,
SelectedBackendID: result.SelectedBackendID,
ModelName: result.ModelName,
Endpoint: result.Endpoint,
EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams),
@@ -131,7 +136,7 @@ func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain
if override == nil {
return nil, nil
}
extraParams, err := copyPublicJSONMap(override.ExtraParams)
extraParams, err := jsonvalue.CopyMap(override.ExtraParams)
if err != nil {
return nil, err
}
@@ -143,7 +148,7 @@ func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain
TopP: copyFloat64Ptr(override.TopP),
TimeoutSeconds: copyIntPtr(override.TimeoutSeconds),
ServiceTier: override.ServiceTier,
ReasoningEffort: override.ReasoningEffort,
ReasoningEffort: copyStringPtr(override.ReasoningEffort),
APIKeyEnv: override.APIKeyEnv,
ExtraParams: extraParams,
}, nil
@@ -151,6 +156,7 @@ func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain
func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
return ExecutionTarget{
BackendID: target.BackendID,
Endpoint: target.Endpoint,
Model: target.Model,
Temperature: target.Temperature,
@@ -396,6 +402,14 @@ func copyFloat64Ptr(src *float64) *float64 {
return &v
}
func copyStringPtr(src *string) *string {
if src == nil {
return nil
}
v := *src
return &v
}
func copyIntPtr(src *int) *int {
if src == nil {
return nil

21
doc.go
View File

@@ -2,9 +2,10 @@
// prompt-defined LLM workflows.
//
// Applications construct an [Engine] with [NewEngine], select filesystem or
// in-memory sources with options, and call [Engine.Prepare] or [Engine.Run].
// Concrete repositories, validators, and the built-in OpenAI-compatible client
// remain internal implementation details.
// in-memory sources and optional engine-scoped [Backend] registrations, and
// call [Engine.Prepare] or [Engine.Run]. Concrete registries, repositories,
// validators, and the built-in OpenAI-compatible client remain internal
// implementation details.
//
// # Concurrency and ownership
//
@@ -12,11 +13,11 @@
// or [ArtifactReader] can therefore receive concurrent calls and must be safe
// for that use.
//
// NewEngine copies in-memory profiles. Prepare and Run copy request maps,
// slices, pointer values, and JSON-compatible extra parameters before using
// them. Returned values and values passed to extension interfaces are likewise
// isolated from engine state. Callers own those copies and may mutate them
// after the call that supplied or returned them.
// NewEngine copies in-memory profiles and backend definitions. Prepare and Run
// copy request maps, slices, pointer values, and JSON-compatible extra
// parameters before using them. Returned values and values passed to extension
// interfaces are likewise isolated from engine state. Callers own those copies
// and may mutate them after the call that supplied or returned them.
//
// # Security and sensitive data
//
@@ -42,8 +43,8 @@
// [GenerateResponse], [ExecutionTargetPresence], and the string value types
// used by those values.
//
// Construction values, including [Config], [RunRequest], [ArtifactRef],
// [ExecutionTargetOverride], [Profile], and
// Construction values, including [Config], [Backend], [RunRequest],
// [ArtifactRef], [ExecutionTargetOverride], [Profile], and
// [OpenAICompatibleProfileConfig], do not have stable JSON representations.
// Direct API keys are nevertheless excluded from JSON for every public value.
//

View File

@@ -33,8 +33,8 @@ engine, err := promptkit.NewEngine(promptkit.Config{
})
```
Options support single-file or `fs.FS` sources, in-memory profiles, and
injected artifact or model clients. Consult the
Options support single-file or `fs.FS` sources, in-memory profiles,
engine-scoped backends, and injected artifact or model clients. Consult the
[constructor and option GoDoc](../../engine.go) for composition, precedence,
validation, and default transport behavior. Source discovery, format
validation, and profile precedence are defined by the
@@ -97,6 +97,66 @@ For programmatic profiles,
[`OpenAICompatibleProfile`](../../profiles.go) converts ordinary
OpenAI-compatible settings into a value accepted by `WithProfiles`.
### Set A Per-Run Session And Reasoning
Supply a direct session ID when one prompt should be correlated with a
consumer-managed conversation or workflow without changing prompt variables:
```go
reasoning := "high"
result, err := engine.Run(ctx, promptkit.RunRequest{
PromptID: "meeting.summary",
SessionID: "conversation-42",
Inputs: map[string]promptkit.ArtifactRef{
"note": promptkit.Inline("Synthetic meeting notes"),
},
Execution: &promptkit.ExecutionTargetOverride{
ReasoningEffort: &reasoning,
},
})
```
A nil reasoning pointer inherits the selected profile, a pointer to a
nonblank string replaces it, and a pointer to a blank string disables
reasoning for that run. Session IDs are correlation metadata, not credentials;
use stable, non-secret values that are safe to expose to collaborators and
providers. The
[`RunRequest` and `ExecutionTargetOverride` GoDoc](../../types.go) owns the
exact normalization, precedence, error, copying, and exposure contract.
### Register A Custom Backend
Register a reusable OpenAI-compatible connection once, then select it from a
profile:
```go
engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: "prompts",
},
promptkit.WithBackend(promptkit.Backend{
ID: "local",
Endpoint: "http://localhost:8000/v1",
APIKeyEnv: "LOCAL_LLM_API_KEY",
}),
promptkit.WithProfiles(promptkit.Profile{
ID: "local-summary",
BackendID: "local",
Model: "example-model",
}),
)
```
Registrations belong to one engine and custom IDs cannot replace built-ins.
The [`Backend` and `WithBackend` GoDoc](../../backends.go) defines validation,
copying, uniqueness, and request-default behavior.
Both file-backed and in-memory profiles select a registration through
`backend` or `Profile.BackendID`. Profile and request endpoint overrides retain
that routing identity. `PreparedRun.SelectedBackendID`,
`RunResult.SelectedBackendID`, and the effective `ExecutionTarget.BackendID`
expose it to consumers and injected model clients. Endpoint-only profiles
remain supported and expose an empty backend ID.
## Credentials
File-backed profiles name an environment variable; in-memory profiles can
@@ -137,7 +197,8 @@ distinguish invalid construction, invalid requests, absent sources,
source-loading failures, collaborator failures, and operational validation
failures. Specific request conditions may also match the broader
`ErrInvalidRequest`, and injected collaborator identities are preserved where
documented.
documented. Invalid or duplicate backend registrations match
`ErrInvalidConfig`; selecting an unknown backend matches `ErrProfileLoad`.
## Application Boundary

View File

@@ -91,7 +91,8 @@ of a named input. Missing variables and input references are errors.
The optional `session_id` uses the same template data and input helper. Its
rendered value is trimmed, omitted when empty, and limited to 256 Unicode code
points.
points. A nonblank direct request session ID bypasses this template completely;
a blank direct value leaves the template behavior unchanged.
### Cache Control
@@ -136,7 +137,7 @@ A profile supplies model execution settings:
```yaml
id: local-summary
endpoint: http://localhost:8000/v1
backend: openrouter
model: example-model
temperature: 0.2
max_tokens: 500
@@ -144,7 +145,6 @@ top_p: 0.95
timeout_seconds: 90
service_tier: flex
reasoning_effort: medium
api_key_env: EXAMPLE_API_KEY
extra_params:
provider_option: enabled
```
@@ -152,7 +152,8 @@ extra_params:
| Field | Required | Meaning |
| --- | --- | --- |
| `id` | yes | Non-empty profile identifier. IDs must be unique within one source. |
| `endpoint` | yes | Non-empty OpenAI-compatible base URL, including an API version path when required. |
| `backend` | unless `endpoint` is present | Backend registry ID. It is trimmed and registry membership is checked when the profile is prepared. |
| `endpoint` | unless `backend` is present | Non-empty OpenAI-compatible base URL, including an API version path when required. When both connection fields are present, this overrides the backend endpoint without changing backend identity. |
| `model` | yes | Non-empty provider model name. |
| `temperature` | no | Number from 0 through 2. |
| `max_tokens` | no | Integer zero or greater. |
@@ -166,6 +167,13 @@ extra_params:
Raw `api_key` is prohibited in profile YAML. Store only an environment
variable name in `api_key_env`.
Promptkit does not infer a backend from a model or endpoint. Endpoint-only
profiles remain supported and have no effective backend ID.
The engine always provides the built-in `openrouter` ID. Consumers can add
engine-scoped IDs with
[`WithBackend`](../backends.go); exact registration validation belongs to its
GoDoc.
`extra_params` accepts null, booleans, finite numbers, strings, arrays, and
objects with string keys. Keys must be non-empty. With the built-in client,
they also cannot collide with the standard fields listed in the
@@ -176,8 +184,9 @@ they also cannot collide with the standard fields listed in the
Execution settings resolve in this order:
1. framework defaults;
2. the selected profile; and
3. request `ExecutionTargetOverride` values.
2. the selected backend, when the profile names one;
3. the selected profile; and
4. request `ExecutionTargetOverride` values.
The framework defaults are:
@@ -194,8 +203,13 @@ explicit zero is preserved. In particular, an explicit request
`timeout_seconds` of zero disables the per-generation deadline while leaving
the caller context and transport timeout intact.
Non-empty request strings replace profile strings. A non-empty request
`ExtraParams` map replaces the profile map rather than merging keys.
Non-empty profile strings replace backend defaults, and non-empty request
strings replace both. Request reasoning is the exception: a nil
`ReasoningEffort` pointer inherits the profile, a pointer to a nonblank string
trims and replaces it, and a pointer to a blank string clears it. Backend
identity is retained when either layer overrides the endpoint. A non-empty
`extra_params` map at each layer replaces the entire lower-precedence map
rather than merging keys.
The [outbound integration contract](integrations/openai-compatible-chat.md)
defines how the effective settings are serialized.
@@ -217,8 +231,11 @@ invalid matching profile is an error and does not fall back. In-memory
## Built-In Profile Catalog
Built-ins use the OpenRouter-compatible endpoint and
`OPENROUTER_API_KEY`. A custom or in-memory profile with the same ID takes
Every built-in selects the `openrouter` backend. The engine's built-in backend
registry supplies `https://openrouter.ai/api/v1` and the environment-variable
name `OPENROUTER_API_KEY`, so individual profiles contain only model and
generation settings. Built-in profile files do not repeat those connection
values. A custom or in-memory profile with the same profile ID takes
precedence.
| Provider | ID | Model |
@@ -270,6 +287,10 @@ prompt, profile, schema, or example files:
- a request can provide a direct `APIKey` or override `APIKeyEnv`; and
- a direct request key takes precedence over environment lookup.
After a direct request key, the credential-source precedence is request
`APIKeyEnv`, profile `api_key_env`, then the backend default. An in-memory
profile with `APIKeyRequired` clears an inherited backend environment name and
requires a direct key unless the request explicitly supplies `APIKeyEnv`.
Promptkit validates required credential availability during preparation.
Direct keys are excluded from JSON results and redacted by public string
formatters. Environment-variable names may appear in prepared metadata, but

View File

@@ -13,10 +13,15 @@ that produce these outbound settings.
## Endpoint And Method
Generation sends an HTTP `POST` with `Content-Type: application/json`.
A non-empty endpoint from the execution target overrides the client's
configured base URL. After trailing slashes are removed,
`/chat/completions` is appended. Generation fails before sending when neither
source supplies an endpoint.
Before the client is called, the engine resolves framework, backend, profile,
and request values into one execution target. A non-empty endpoint from that
target overrides the client's configured base URL. After trailing slashes are
removed, `/chat/completions` is appended. Generation fails before sending when
neither source supplies an endpoint.
The target's backend ID is routing metadata for prepared values, results, and
injected clients. The built-in client does not derive the URL from that ID and
does not serialize it in the provider request.
## Authentication
@@ -26,6 +31,12 @@ the client reads that variable and requires a non-empty value. The selected
key is sent as `Authorization: Bearer <key>`. No authorization header is sent
when neither mechanism is configured.
The target contains the already resolved environment-variable name: an
explicit request override takes precedence over profile metadata, which takes
precedence over the backend default. Only the name reaches prepared metadata;
the environment value is read just before the provider call and is never added
to the JSON body.
## Request Body
The request body always contains `model` and `messages`. The execution
@@ -36,20 +47,24 @@ Each ordinary message contains its `role` and string `content`. A
cache-controlled message instead uses a text content block containing `type`,
`text`, and `cache_control`; an empty cache-control TTL is omitted.
A non-empty session ID is trimmed, checked against the internal domain limit,
and sent as top-level `session_id`. It is not sent as a session header.
The effective direct or prompt-rendered session ID is trimmed, limited to 256
Unicode code points, and sent when nonempty as top-level `session_id`. It is
never also sent as a session header.
The client conditionally includes:
- `temperature`, `max_tokens`, and `top_p` when non-zero or explicitly
present;
- non-empty `service_tier` and `reasoning_effort`; and
- non-empty `service_tier` and effective `reasoning_effort`; an explicitly
disabled reasoning setting is empty and therefore omitted; and
- `response_format` for JSON Schema structured output, including its name,
strict flag, and schema document.
Extra parameters are merged directly into the top-level body after JSON
serialization is verified. Empty keys and collisions with these reserved
fields are rejected before any provider call:
The engine resolves backend, profile, and request extra-parameter maps by
whole-map replacement rather than key merging. The resulting effective map is
then merged directly into the top-level body after JSON serialization is
verified. Empty keys and collisions with these reserved fields are rejected
before any provider call:
- `model`
- `session_id`
@@ -61,6 +76,9 @@ fields are rejected before any provider call:
- `reasoning_effort`
- `response_format`
`backend_id`, `api_key_env`, and resolved credential values are not provider
request fields.
## Response Handling
Any 2xx response is decoded as an OpenAI-compatible chat response. The client

View File

@@ -20,6 +20,11 @@ orchestration. `OpenAICompatibleClient` is the built-in implementation. It
uses internal domain values for rendered prompts, execution targets,
structured output, responses, and token usage.
The runner supplies a fully resolved target after applying backend, profile,
and request precedence. The client uses its endpoint, credential metadata,
generation fields, and extra parameters. `BackendID` remains routing metadata
for the generation boundary and is not mapped into the provider payload.
Construction validates the configured base URL and clones any supplied
`http.Client` so Promptkit can apply its timeout default without mutating the
caller's client. Generation then:
@@ -31,6 +36,10 @@ caller's client. Generation then:
5. performs the outbound request under the applicable deadlines; and
6. decodes the first response choice and token usage.
`internal/llm` owns the set of reserved OpenAI-compatible request fields used
when validating extra parameters. Backend registration consumes the same rule
without making the model client depend on registry configuration.
The implementation has no retry loop, tool-call support, provider catalog,
inbound HTTP behavior, or durable session store.
@@ -51,5 +60,7 @@ The
[OpenAI-compatible client tests](../../internal/llm/openai_compatible_client_test.go)
own configuration, client cloning, deterministic deadline precedence,
authentication, request and response mapping, malformed data, error identity,
cancellation, and response-body suppression. They use local test servers and
test transports; the default suite makes no live or paid provider requests.
cancellation, and response-body suppression. The root transport contract test
also verifies that resolved backend settings reach this client without
serializing backend identity. All use local test servers or test transports;
the default suite makes no live or paid provider requests.

View File

@@ -11,20 +11,22 @@ contributor workflow and validation.
| Component | Implemented responsibility | References |
| --- | --- | --- |
| Root `promptkit` package | Provides the supported engine facade, source and injection options, public request and result values, built-in profile construction, extension interfaces, value conversion, redacted formatting, and public error mapping. | [Package GoDoc](../../doc.go), [engine assembly](../../engine.go) |
| Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request and result values, profile construction, extension interfaces, value conversion, redacted formatting, and public error mapping. | [Package GoDoc](../../doc.go), [backend API](../../backends.go), [engine assembly](../../engine.go) |
| `examples/go-library/prepare` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, and `Prepare`. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
| `internal/backend` | Constructs each engine's immutable registry from the built-in OpenRouter definition and consumer additions, validates and defensively copies definitions through the shared JSON-value package, and consumes the LLM-owned OpenAI-compatible reserved request-field rule. | [Backend registry](../../internal/backend/registry.go) |
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation. | [Domain declarations](../../internal/domain/domain.go) |
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
| `internal/jsonvalue` | Validates and deeply copies JSON-compatible extra-parameter trees while preserving supported concrete value types. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
| `internal/promptdef` | Loads strictly decoded, validated prompt definitions from filesystem and `fs.FS` sources, including version selection and contained file-backed message content. | [Framework formats](../formats.md), [prompt-definition repository](../../internal/promptdef/filesystem_repository.go) |
| `internal/profile` | Loads strictly decoded, validated execution profiles from filesystem and `fs.FS` sources and composes repositories with error-preserving fallback. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go) |
| `internal/profile/builtin` | Embeds the built-in execution profile catalog and combines it with an optional primary repository. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) |
| `internal/profile` | Loads strictly decoded, validated execution profiles, including backend selection, from filesystem and `fs.FS` sources and composes repositories with error-preserving fallback. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go) |
| `internal/profile/builtin` | Embeds the built-in profile catalog, whose entries select OpenRouter, and combines it with an optional primary repository. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) |
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests, response decoding, authentication, and deadline handling. | [Internal model client](llm.md) |
| `internal/usecase` | Coordinates preparation and execution across internal sources, rendering, artifact loading, generation, validation, and optional repair. | [Internal runner](runner.md) |
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
| `internal/usecase` | Resolves backend, profile, and request settings and coordinates preparation and execution across internal sources, rendering, artifact loading, generation, validation, and optional repair. | [Internal runner](runner.md) |
The root package assembles these internal components without exposing their
representations. Consumers depend only on the root facade.

View File

@@ -17,10 +17,12 @@ and override semantics consumed by the runner.
## Collaborators
`Runner` coordinates narrow internal interfaces for prompt definitions,
profiles, artifacts, rendering, model generation, and validation. Schema
documents are loaded through the validator's optional schema-loader interface.
An output repairer can be injected internally, but the ordinary runner
constructor does not enable one.
profiles, backend resolution, artifacts, rendering, model generation, and
validation. The root engine supplies one immutable registry containing the
built-in backend and validated consumer additions. Schema documents are loaded
through the validator's optional schema-loader interface. An output repairer
can be injected internally, but the ordinary runner constructor does not
enable one.
Each invocation carries its state in request, prepared-run, and result values.
The runner has no durable run or session store.
@@ -29,23 +31,42 @@ The runner has no durable run or session store.
`Prepare` performs the reusable pre-generation workflow:
1. validate the prompt selection and load the prompt definition;
2. hash the loaded definition;
1. validate the required prompt selection and normalize any direct session ID;
2. load the prompt definition and hash the original definition;
3. select the request profile or the prompt's default profile;
4. resolve application-neutral defaults, profile values, and explicit request
overrides in that order;
5. validate endpoint, model, numeric overrides, and credential requirements;
6. resolve the output contract and load a structured-output schema when
4. resolve the profile's backend ID, when present;
5. resolve application-neutral defaults, backend defaults, profile values,
and explicit request overrides in that order;
6. validate endpoint, model, numeric overrides, and credential requirements;
7. resolve the output contract and load a structured-output schema when
required;
7. load and hash input artifacts;
8. render and hash the prompt; and
9. return the effective settings, source identities, messages, hashes, and
8. load and hash input artifacts;
9. render messages, resolve the effective session ID, and hash the effective
rendered prompt; and
10. return the effective settings, source identities, messages, hashes, and
preparation timing.
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
out-of-range values fail as invalid requests. A direct API key takes
precedence over environment lookup for execution; secret values remain
excluded from serialized metadata.
out-of-range values fail as invalid requests. Endpoint overrides do not change
the selected backend identity. Non-empty extra-parameter maps replace whole
lower-precedence maps. A direct API key takes precedence over environment
lookup; otherwise request, profile, and backend environment-variable names
apply in that order. A profile requiring a direct key clears an inherited
backend environment name unless the request supplies its own name. Secret
values remain excluded from serialized metadata.
Reasoning overrides are tri-state: nil inherits the profile, a pointer to a
nonblank string trims and replaces it, and a pointer to a blank string clears
it. A nonblank direct session is normalized before source loading, bypasses
the prompt session template, and is applied after ordinary message rendering.
A blank direct value retains prompt-template behavior. The runner clears the
template only on a value copy of the definition, so the definition hash always
describes the original source while the rendered-prompt hash includes the
effective direct or rendered session.
The registry is read-only after engine construction. Concurrent `Prepare` and
`Run` calls resolve independent defensive backend values and keep all
invocation state local.
## Run Flow
@@ -56,12 +77,16 @@ an inability to perform validation is an operational error.
When an internal repairer is present, a JSON or JSON Schema content failure can
trigger bounded repair attempts. Repair receives the effective execution
target, validation errors, prior output, and structured-output specification.
This capability remains internal and is not a public option.
target and session ID, validation errors, prior output, and structured-output
specification. This capability remains internal and is not a public option.
A successful result includes the output artifact and raw output, validation
state, prompt and rendered-prompt hashes, selected profile, effective settings,
input hashes, token usage, a generated run identifier, and UTC timing.
state, effective session ID, prompt and rendered-prompt hashes, selected
profile and backend, effective settings, input hashes, token usage, a generated
run identifier, and UTC timing. The same effective session reaches initial
generation and any repair attempt through the rendered prompt. The same
effective target, including backend identity, reaches generation and any
repair attempt.
## Failure Categories
@@ -71,12 +96,17 @@ validation failures. Wrapping preserves the package identities mapped by the
public facade and retains collaborator identities where they are part of the
internal contract. Context cancellation propagates through the invoked
collaborator and is classified by the owning operation.
An overlong direct session is an invalid request before source loading, while
an invalid or overlong prompt session template remains a prompt-render failure.
An unknown selected backend, or a selected backend with no configured resolver,
is classified as a profile-load failure.
## Test Ownership And Changes
The [runner tests](../../internal/usecase/runner_test.go) own preparation order,
selection and override precedence, schema-before-generation behavior, hashing,
generation and validation outcomes, bounded repair, credentials and redaction,
selection and override precedence, direct-session resolution,
schema-before-generation behavior, hashing, generation and validation
outcomes, backend propagation, bounded repair, credentials and redaction,
error categories, artifact metadata, usage, and timing.
Changes to orchestration should continue to use the existing package

View File

@@ -24,13 +24,20 @@ duplicate detection, and source containment:
`internal/profile` loads and validates execution profiles from an
operating-system filesystem or an `fs.FS`. It supports a primary repository
with fallback only when the primary reports that a profile is absent.
with fallback only when the primary reports that a profile is absent. Strict
YAML decoding recognizes the optional `backend` field, trims its value, and
requires a model plus at least one non-blank backend or endpoint. Loading does
not check registry membership because the available registry belongs to the
assembled engine; the runner checks membership during preparation.
`internal/profile/builtin` embeds the maintained built-in profile catalog and
can place a caller-selected repository ahead of that catalog. Profile behavior
is owned by the
can place a caller-selected repository ahead of that catalog. Every embedded
profile selects `openrouter` and inherits its endpoint and credential
environment-variable name from the built-in backend registry rather than
repeating those values. Profile behavior is owned by the
[profile repository tests](../../internal/profile/repository_test.go), while
catalog completeness, duplicate IDs, and overlay behavior are owned by the
catalog completeness, the backend-selection invariant, duplicate IDs, and
overlay behavior are owned by the
[built-in repository tests](../../internal/profile/builtin/repository_test.go).
## Ordinary Artifacts

View File

@@ -21,10 +21,14 @@ The implemented internal components consist of:
- `internal/domain`, which owns framework data values shared by later internal
components;
- `internal/backend`, which owns validated immutable OpenAI-compatible backend
definitions and the built-in OpenRouter definition;
- `internal/defaults`, which owns application-neutral framework defaults and
constructs the default execution target;
- `internal/filecatalog`, which discovers YAML files and provides source-path
helpers for filesystem and `fs.FS` consumers;
- `internal/jsonvalue`, which validates and defensively copies JSON-compatible
extra-parameter trees;
- `internal/promptdef`, which loads and validates prompt definitions from
filesystem and `fs.FS` sources;
- `internal/profile`, which loads, validates, and overlays execution profiles
@@ -45,17 +49,19 @@ The `examples/go-library/prepare` and `examples/go-library/run` packages are
maintained downstream consumers of the root facade. They do not expose library
packages or participate in internal assembly.
The root facade assembles the internal repositories, renderer, validator,
outbound client, and use-case runner while translating public values and
errors at the library boundary. The defaults and renderer depend on the domain
model. Prompt-definition and profile repositories use the domain model, file
catalog, and YAML decoder. The built-in profile repository supplies an
embedded `fs.FS` to the profile package. Artifact reading uses the domain model
and application-neutral defaults. Validation uses the domain model, file
catalog, and JSON Schema implementation. The model client uses the domain
model, application-neutral defaults, and an injected or standard-library HTTP
client. The use-case runner depends on the narrow interfaces owned by each
internal component.
The root facade assembles one immutable backend registry, the internal
repositories, renderer, validator, outbound client, and use-case runner while
translating public values and errors at the library boundary. The registry
contains built-ins plus validated engine-scoped consumer additions. The
defaults and renderer depend on the domain model. Prompt-definition and
profile repositories use the domain model, file catalog, and YAML decoder. The
built-in profile repository supplies an embedded `fs.FS` to the profile
package. Artifact reading uses the domain model and application-neutral
defaults. Validation uses the domain model, file catalog, and JSON Schema
implementation. The model client uses the domain model, application-neutral
defaults, and an injected or standard-library HTTP client. The use-case runner
depends on the narrow interfaces owned by each internal component, including
backend lookup.
The current implementation follows this dependency direction:
@@ -72,9 +78,13 @@ downstream consumers, including Scriptorium
narrow injected abstractions
```
The facade coordinates internal components and adapts the supported public
extension interfaces to narrow internal abstractions. Internal components must
not depend on consumers or on Scriptorium.
The backend registry depends on the domain model and shared JSON-value
validation, has no mutation API after construction, and consumes the
OpenAI-compatible reserved request-field rule owned by the model client. The
model client does not depend on registry configuration. The facade coordinates
internal components and adapts
the supported public extension interfaces to narrow internal abstractions.
Internal components must not depend on consumers or on Scriptorium.
## Repository And Consumer Boundary

236
docs/roadmap/concurrency.md Normal file
View File

@@ -0,0 +1,236 @@
# Backend-Specific Concurrency Management
**Status:** Accepted.
## Purpose
This roadmap defines the scope and target end state for engine-local,
backend-specific concurrency management. It records the intended capability,
consumer value, and important policy choices.
This document is planning material, not a description of current behavior.
Current exported contracts remain owned by Go declarations and GoDoc, backend
registration guidance by the
[consumer guide](../consumers/pkg-promptkit.md#register-a-custom-backend), and
implemented orchestration by the
[internal runner document](../internal/runner.md).
## Motivation
Different model backends can sustain very different request loads. A local
network endpoint may need a small concurrency limit, while OpenRouter can
usually accept substantially more simultaneous work. Requiring every consumer
to build its own semaphores and queues would duplicate routing knowledge,
create inconsistent cancellation behavior, and make it easy for one caller to
bypass the intended backend limit.
Promptkit should own this coordination because it already resolves each run to
an engine-scoped backend identity and owns every model-generation call made by
the runner. Consumers should continue submitting ready-to-run requests through
the synchronous API, including concurrently from multiple goroutines, without
implementing their own backend scheduler.
The buffered queue is a safety boundary, not an ordinary throughput
restriction. Its primary purpose is to prevent a bug or unintended submission
loop from creating an unbounded in-memory backlog.
## Scope
The feature will add optional concurrency policy to registered backends and
coordinate `Run` calls against independent per-backend capacity pools.
Each policy has two distinct controls:
- an active-generation limit, which protects the backend from too many
simultaneous model requests; and
- a bounded waiting capacity, which protects the process from admitting an
unbounded backlog.
Concurrency policy belongs to a backend registration. It is not a profile
model parameter and cannot be overridden per run. Profiles select the policy
through their backend ID, while a profile or request endpoint override remains
in the selected backend's pool.
`Prepare` does not call a model and will remain outside concurrency admission.
## Defaults And Configuration
The built-in OpenRouter backend will use:
- an active-generation limit of 16; and
- a waiting capacity of 1024.
The waiting default is intentionally generous. Reaching it should indicate
abnormal submission pressure rather than normal application behavior.
Consumer-registered backends will remain unlimited unless the consumer
configures an active-generation limit. When a consumer enables a limit and
does not specify waiting capacity, the waiting capacity will default to 1024.
Consumers may configure a different bounded capacity, including zero when
they want no admitted backlog beyond the active-limit-sized run set.
The public representation must distinguish an omitted waiting capacity from
an explicit zero.
Endpoint-only profiles have no backend registration from which to obtain
policy and will remain unlimited. A future engine-wide or endpoint-keyed
policy can be considered separately if consumers demonstrate that need.
Invalid limits or capacities will fail engine construction as invalid
configuration. Policy values will be copied into engine-owned immutable state
along with the rest of the backend registration.
## Admission And Execution Behavior
`Run` remains a synchronous, wait-for-result operation. Concurrent callers may
block inside `Run` while waiting for their selected backend, then receive the
ordinary result or error from that invocation.
For a configured pool, the active-generation limit plus the waiting capacity
defines the maximum number of concurrent `Run` invocations that Promptkit will
accept for that backend. A waiting capacity of zero therefore accepts no more
runs than the active limit. Admission is immediate: a call either reserves one
of those bounded slots or receives the capacity error. An accepted run may
then wait internally for active-generation capacity.
For a limited backend, Promptkit will bound the number of accepted runs before
expensive artifact loading, prompt rendering, and large defensive copies where
practical. Lightweight prompt, profile, and backend resolution may occur first
when it is required to identify the correct capacity pool. This pre-admission
resolution must not become a second execution-precedence path with behavior
that can drift from `Prepare`.
An accepted run retains its admission until it completes or fails. Every
actual model-generation call for that run must separately observe the
backend's active-generation limit. This includes:
- the initial generation;
- every output-repair generation; and
- calls made through either the built-in or an injected model client.
Preparation and output validation should not hold an active-generation permit.
A repair remains part of its already-admitted run, but reacquires active
generation capacity so repairs cannot exceed the backend limit. It must not be
rejected merely because new runs filled the waiting queue after its initial
generation.
Within one backend pool, waiting generation calls should be served in FIFO
order, subject to canceled calls being removed. Different backend pools make
progress independently; a saturated local backend must not consume
OpenRouter's active or waiting capacity.
The feature will not promise ordering across backend pools or completion order
among admitted runs.
## Capacity Failure And Cancellation
When a backend's bounded waiting capacity is full, a new `Run` call will fail
promptly rather than waiting outside the bounded admission system. The public
API will expose a recognizable capacity-exhaustion error identity distinct
from invalid configuration, invalid requests, and model-client failures.
Rejected calls return no partial result and do not invoke the model client.
Waiting within the admitted backlog or for active-generation capacity must
honor the caller's context. Cancellation or deadline expiry will:
- stop waiting promptly;
- release any admission or generation capacity held by that invocation;
- preserve the applicable context error identity; and
- avoid invoking the model client if cancellation wins before generation
starts.
Capacity must also be released after preparation, generation, validation,
repair, or collaborator failure. One failed or canceled run must not reduce
the backend's future usable capacity.
Elapsed `Run` timing will include time spent waiting after the call is
accepted. `PreparedRun` timing will continue to describe preparation rather
than queue waiting.
## Engine And Client Boundaries
All pools and queued state belong to one `Engine`. Separate engines do not
share capacity, even when they register the same backend ID or endpoint. The
feature introduces no process-global scheduler.
The engine will apply policy consistently to the built-in model client and an
injected `LLMClient`. Consumers calling their own client outside Promptkit are
outside this boundary. Injected clients remain responsible for their internal
thread safety and cancellation behavior.
Backend policy is keyed by the resolved backend ID rather than endpoint text.
This preserves stable routing when a selected backend's endpoint is overridden
and avoids accidentally combining unrelated registrations that happen to use
the same URL.
## Queue Lifetime And Observability
Admission state is buffered, ephemeral, and in-process. It is not persisted
and has no survival guarantee across engine disposal or process termination.
Promptkit will not introduce background job ownership or require consumers to
start or stop workers.
The initial feature does not require public queue-depth metrics, callbacks, or
inspection APIs. Capacity errors and ordinary call timing provide the
consumer-visible behavior. Operational observability can be added later
without coupling the scheduling mechanism to an application logging or
metrics system.
## Compatibility
Consumer-registered backends and endpoint-only profiles remain unlimited
unless concurrency is explicitly configured, preserving their existing
behavior.
The built-in OpenRouter backend will change from unlimited concurrency to a
limit of 16 with a bounded waiting capacity of 1024. Ordinary synchronous
calls remain unchanged, while unusually high concurrent use may now wait or
return the capacity error. This behavioral change must be identified in the
release notes for the version that publishes it.
Adding backend policy fields and a public capacity error is otherwise
additive. The change will use a pre-`v1` minor release under Promptkit's
[release policy](../release.md#release-model).
## Non-Goals
This scope does not include:
- asynchronous job handles, polling, or detached result delivery;
- durable or cross-process queues;
- persistence or recovery across engine or process shutdown;
- priorities, scheduling weights, or consumer-defined fairness classes;
- automatic retries, backoff, rate-limit interpretation, or provider quota
discovery;
- token-per-minute or request-per-minute rate limiting;
- dynamic reconfiguration after engine construction;
- per-profile or per-run concurrency overrides;
- endpoint-keyed pooling for profiles without a backend ID;
- process-global coordination across engines;
- application worker lifecycle, logging, tracing, or metrics policy; or
- changes to prompt, profile, schema, or model-provider wire formats.
## Target End State
This roadmap reaches its target end state when:
- each engine independently coordinates configured backend capacity;
- the built-in OpenRouter backend allows 16 active generations and up to 1024
waiting runs;
- consumer backends can opt into their own active and waiting limits while
remaining unlimited by default;
- endpoint overrides retain the selected backend's capacity pool and
endpoint-only profiles remain unlimited;
- synchronous `Run` callers wait for and receive their ordinary result;
- admission is bounded before expensive preparation work where practical;
- every initial and repair generation observes the backend's active limit
without serializing preparation or validation;
- a full waiting queue returns a recognizable capacity error without invoking
the model client;
- cancellation and all failure paths promptly release capacity and preserve
context error identity;
- built-in and injected model clients receive the same scheduling behavior;
- pools remain ephemeral, engine-scoped, and independent across backend IDs;
and
- current-state GoDoc, consumer, internal, and release documentation describe
the implemented behavior once it lands.

View File

@@ -1,222 +0,0 @@
# Documentation Hardening Roadmap
## Purpose
This roadmap coordinates a focused pass over Promptkit's documentation and
public contract. The work should close the gaps identified by the documentation
audit, strengthen canonical ownership, and leave consumers and contributors
with guidance that is accurate, navigable, and proportionate to their needs.
This is planning material, not a description of implemented behavior. Follow
the [documentation policy](../policy/documentation.md) throughout the work and
update current-state documents only when their claims are supported by the
implementation and tests.
## Scope And Principles
The work covers public GoDoc, consumer guidance, maintained examples, format
and integration references, architecture ownership, roadmap lifecycle, and
documentation validation.
- Resolve ambiguous behavior before documenting it as a contract.
- Keep exact exported API semantics in Go declarations and GoDoc.
- Keep task-oriented guidance in consumer documents and complete runnable
artifacts in `examples/`.
- Put security-relevant consumer responsibilities at the public boundary, not
only in internal contributor documents.
- Remove duplicated ownership instead of synchronizing parallel references.
- Make documentation checks reproducible where practical.
- Preserve existing behavior unless a stage explicitly selects and tests an
API change.
The roadmap does not add new Promptkit features, redesign framework formats,
or implement ideas from [the future feature catalog](future.md). If resolving
an ambiguity requires a behavioral change, treat that change as a separately
reviewable implementation unit and update its canonical documentation in the
same unit.
## Stage 1: Resolve Public Contract Questions
Before expanding prose, decide the intended contract for exported behavior
that is currently ambiguous.
- [x] Decide whether `RunRequest.Metadata` has a supported observable purpose.
Define its propagation and ownership, or remove or deprecate it through an
intentional public API change.
- [x] Decide and test whether one `Engine` supports concurrent `Prepare` and
`Run` calls.
- [x] Decide how repeated options of the same category behave, including
prompt, profile, schema, model-client, and artifact-reader options.
- [x] Define which public values have supported JSON representations.
- [x] Define JSON time units and omission behavior, including the relationship
between prepared-run and run-result durations.
- [x] Decide whether run IDs and exposed hashes have stable formats or must be
treated as opaque values.
- [x] Confirm the intended transport-timeout default and its zero or negative
configuration semantics.
- [x] Confirm the supported JSON Schema dialect and reference boundaries,
including whether remote references are allowed.
The selected exported API contracts are implemented and tested. Their durable
definitions now belong to the root package declarations and GoDoc.
One format-level decision remains here until Stage 5 moves it to the framework
format reference: JSON Schema uses Draft 2020-12, with that dialect selected
when `$schema` is omitted. Same-document fragments and relative references
contained by a directory or `fs.FS` schema root are supported. A single-file
source supports only references contained in that document. Absolute,
escaping, and remote references are rejected.
**Gate:** Each question has an explicit answer backed by existing behavior or
by an accepted implementation change and proportionate tests. No later stage
should invent a contract merely to fill a documentation gap.
## Stage 2: Make GoDoc The Canonical Public Contract
Strengthen the root package declarations so `go doc` is sufficient to
understand exact public behavior without relying on internal documents.
- [x] Add useful field-level GoDoc to configuration, request, profile,
execution-target, result, artifact, validation, structured-output, and model
client values.
- [x] Document required fields and nil, empty, and zero-value semantics.
- [x] Document override, replacement, profile-precedence, and copy-ownership
behavior where it belongs to the exported API.
- [x] Document credential inputs, redaction, and the values intentionally
excluded from serialization.
- [x] Give each public error sentinel an accurate comment and document the
supported `errors.Is` relationships.
- [x] Document engine concurrency and option-composition behavior selected in
Stage 1.
- [x] Document serialization, time, run-ID, and hash semantics selected in
Stage 1.
- [x] Review constructor, option, extension-interface, `Prepare`, and `Run`
GoDoc for complete failure and cancellation expectations.
Update the [consumer guide](../consumers/pkg-promptkit.md) to summarize and link
to these contracts instead of maintaining exhaustive copies of exported names
or exact semantics.
**Gate:** `go doc -all .` presents a coherent public contract, exported
declarations have accurate comments, and contract tests protect every newly
documented behavior whose compatibility risk warrants durable coverage.
## Stage 3: Improve Consumer Safety And Executable Guidance
Move consumer-relevant security boundaries to the places where consumers will
encounter them and add one representative execution workflow.
- [x] Explain in public GoDoc and the consumer guide that the default file
artifact reader accepts unrestricted caller-selected paths.
- [x] Make clear that Promptkit does not impose an application root, inbound
request-size policy, or untrusted-input security boundary.
- [x] Explain that rendered messages, artifact bodies, raw model output, and
validation details may be sensitive even when credentials are redacted.
- [x] Clarify the responsibilities of injected artifact readers and model
clients for cancellation, copying, logging, and secret handling.
- [x] Add a maintained offline `Run` example using an injected deterministic
model client, without credentials, live network access, or paid calls.
- [x] Link the consumer guide to the execution example and keep embedded
snippets smaller than the maintained artifact.
- [x] Decide whether the existing preparation example should remain separate
or share reusable fixtures without obscuring either workflow.
The preparation and execution examples remain separate, self-contained
workflows. Each keeps its own small prompt fixture so consumers can copy or run
one example without depending on the other.
**Gate:** Both preparation and execution have complete, secret-free,
deterministic consumer examples, and the consumer guide exposes the important
filesystem and data-sensitivity boundaries without leaking internal mechanics.
## Stage 4: Restore Canonical Ownership
Remove parallel definitions and make navigation follow the ownership model in
the documentation policy.
- [ ] Reduce the [architecture policy](../policy/architecture.md) to durable
boundaries, layers, dependency direction, invariants, and non-goals.
- [ ] Keep the exact implemented package and component inventory solely in the
[internal component overview](../internal/overview.md).
- [ ] Review the consumer guide's public error and option lists so they remain
task-oriented summaries rather than duplicate API references.
- [ ] Review internal documents for public-contract statements that should be
links to GoDoc or the format and integration owners.
- [ ] Reconcile the documentation policy's temporary-roadmap lifecycle with
the continuing idea-catalog role of `docs/roadmap/future.md`.
- [ ] Rephrase or link roadmap statements that depend on exact current API
behavior, particularly the runtime reasoning entry.
- [ ] Confirm that every document states its audience or purpose and links to
the canonical owner of adjacent topics.
**Gate:** Every authoritative fact has one clear owner, package inventory
changes no longer require edits to the architecture policy, and roadmaps
cannot be mistaken for current-state references.
## Stage 5: Refine Format And Integration References
Close compatibility gaps in the documents that own file formats and outbound
wire behavior.
- [ ] State or canonically link the exact session-ID limit enforced by the
OpenAI-compatible client.
- [ ] State the configured and default transport-timeout behavior without
referring to an unnamed internal default.
- [ ] Document the JSON Schema dialect and local, contained, and remote
reference behavior selected in Stage 1.
- [ ] Clarify structured-output naming and strictness when those values are
part of the public or integration contract.
- [ ] Add a caveat that built-in profiles are maintained configurations, not a
guarantee of continuing third-party model availability.
- [ ] Recheck every prompt, profile, schema, credential, request-body, response,
timeout, and precedence statement against its owning implementation and
tests.
**Gate:** A consumer can determine the supported file and wire compatibility
boundaries without consulting internal source code or relying on unspecified
defaults.
## Stage 6: Make Documentation Validation Reproducible
Align contributor and release procedures around a small, consistent set of
checks.
- [ ] Use one robust command for checking every tracked Go file with `gofmt`.
- [ ] Provide a repository-local or clearly documented command that validates
local Markdown targets and heading fragments.
- [ ] Decide how published external links are checked without making ordinary
validation depend on mutable network services.
- [ ] Reconcile the validation descriptions in the
[development guide](../development.md),
[testing policy](../policy/testing.md), and
[release procedure](../release.md) so one document owns each requirement.
- [ ] Ensure example validation covers every maintained example added by this
roadmap.
- [ ] Keep documentation-only validation proportionate while requiring full Go
validation when commands, examples, generated output, or checked behavior
changes.
**Gate:** A maintainer can run the documented formatting, link, example, Go,
and repository-hygiene checks exactly as written, with no hidden manual
procedure for local documentation.
## Completion Criteria
The roadmap is complete when:
- all Stage 1 contract questions are resolved;
- GoDoc is the authoritative and sufficient exported API reference;
- consumer guidance covers unrestricted file access and sensitive generated
data;
- maintained offline examples cover both `Prepare` and `Run`;
- architecture, inventory, consumer, internal, format, integration, and
roadmap documents follow their assigned ownership boundaries;
- schema, session, timeout, structured-output, and built-in-profile
compatibility statements are explicit;
- documentation validation is reproducible and consistent across contributor
and release workflows; and
- the complete maintainer validation passes.
After completion, move any durable decisions to GoDoc, policy, format,
integration, or ADR owners as appropriate. Remove this roadmap after incoming
links are updated; do not retain it as a second current-state reference.

View File

@@ -33,78 +33,9 @@ consumers.
## Ideas
### Extensible LLM backend registry
Introduce a registry that separates backend-specific connection,
authentication, and limited request defaults from model execution profiles.
Initial support would cover OpenAI-compatible backends and include a small
built-in catalog, potentially starting with OpenRouter. A profile could select
a backend while optionally overriding its default endpoint, and each backend
could name an optional environment variable for its API key without storing
the credential itself. Downstream consumers could register additional,
uniquely named backends, such as OpenAI or unauthenticated local-network
services, but could not replace built-in IDs. Model selection and generation
settings would remain profile concerns, and custom model clients would remain
available for behavior outside the registry's supported protocol.
### Backend-specific concurrency management
Extend the proposed LLM backend registry with optional per-backend concurrency
limits and bounded, buffered admission queues. Promptkit could then route
simultaneous generation requests according to backend capacity while
containing accidental runaway submission. Downstream consumers would continue
invoking synchronous `Run` calls, including concurrently from multiple
goroutines, and each admitted call would wait for and return its ordinary
result.
- Scope limits to an engine instance rather than hidden process-global state.
- Give different backend IDs independent capacity pools. A profile endpoint
override would remain part of its selected backend's pool.
- Configure active concurrency and waiting capacity separately. Concurrency
protects the backend, while queue capacity protects the process from
admitting an unbounded backlog.
- Give queue capacity a generous, configurable bounded default intended as a
safety ceiling for bugs or unintended loops rather than a routine
application constraint. Select an exact default during implementation
planning and measurement.
- Reject a call with a recognizable capacity error when its backend queue is
full rather than allowing it to wait outside the bounded queue.
- Admit requests before expensive preparation and artifact copying where
practical so queued work remains lightweight.
- Apply a limit to each actual generation request, including repair attempts,
without unnecessarily serializing prompt preparation.
- Make queued and active waits respect caller cancellation and deadlines.
- Treat concurrency as backend policy rather than a profile-level model
setting.
- Keep the queue ephemeral and in-process, with no survival guarantee across
engine or process shutdown.
- Preserve the existing execution model as far as practical. Durable jobs,
polling, priorities, application worker lifecycle, retries, and
cross-process coordination would be separate future capabilities.
### Explicit per-run session and reasoning controls
Allow consumers to associate a session ID with each run and to inherit,
replace, or explicitly disable the reasoning effort configured by its selected
profile. Prompt definitions can currently derive a session ID from a template,
and a non-empty runtime `ReasoningEffort` can replace the profile value, but
there is no direct request-level session ID and an empty reasoning value means
that no override was supplied. These controls would let consumers reuse one
prompt and model profile across sessions and reasoning levels without
maintaining duplicate definitions.
- Preserve a prompt's session ID template and a profile's reasoning effort as
reusable defaults.
- Let a directly supplied per-run session ID take precedence over a rendered
prompt default while retaining the existing validation limit and outbound
representation.
- Distinguish an omitted runtime reasoning choice from an explicit request to
disable reasoning.
- Ensure disabling reasoning omits the corresponding provider request setting
rather than relying on a provider-specific magic value.
- Keep the effective session ID and reasoning choice visible in prepared and
run metadata and available to injected model clients without introducing
additional prompt or profile selection mechanisms.
No ideas are currently cataloged. Backend-specific concurrency management has
been selected for active planning in the
[focused concurrency roadmap](concurrency.md).
## Entry Format

View File

@@ -0,0 +1,828 @@
# Backend-Specific Concurrency Management Implementation Plan
**Status:** Ready for implementation.
## Purpose
This document is the decision-complete implementation plan for
[backend-specific concurrency management](concurrency.md). It is written for a
coding agent that will implement each stage in order.
The feature roadmap owns the intended capability, consumer value, policy
choices, compatibility decision, and target end state. This document owns the
concrete API, internal representation, scheduling architecture, implementation
sequence, test ownership, documentation updates, and completion gates.
## Implementation Rules
- Complete the stages in order. Keep the repository compiling and the focused
tests passing at every stage boundary.
- Preserve unrelated working-tree changes. In particular, `concurrency.md` and
the removal of its source idea from `future.md` may already be uncommitted
when implementation begins; retain both.
- Follow every policy under `docs/policy/`, the task-specific reading guide in
`docs/development.md`, and the target behavior in `concurrency.md`.
- Keep the public API in the root `promptkit` package and implementation
details under `internal/`. Do not expose scheduler types or create another
public package.
- Use only the Go standard library for scheduling. Do not add a queue,
semaphore, worker-pool, or metrics dependency.
- Preserve synchronous, wait-for-result `Run`, unrestricted `Prepare`,
engine-local state, endpoint-only profiles, backend-selected profiles,
backend identity through endpoint overrides, and injected `LLMClient`
behavior.
- Do not broaden the work into asynchronous jobs, durable queues, retries,
rate limiting, dynamic configuration, priorities, worker lifecycle,
endpoint-keyed pools, or public queue observability.
- Keep all tests deterministic, bounded, offline, and race-safe. Coordinate
concurrent tests with channels and barriers rather than timing assumptions
or live providers.
- Update exact GoDoc with each exported declaration change. Update durable
current-state documents only after the corresponding behavior is
implemented.
- Test configurable mechanisms with small test-owned limits. Assert the exact
OpenRouter `16` and default queue `1024` values only at the registry contract
that owns those operational defaults.
- Do not create a release, change a module version, or tag a commit. The final
implementation handoff must identify the built-in OpenRouter behavior change
for the next pre-`v1` minor release.
## Fixed Design
### Public Backend Configuration
Append these fields to the existing root `Backend` type in `backends.go`:
```go
type Backend struct {
// Existing fields remain unchanged and in their current order.
ConcurrencyLimit int
QueueCapacity *int
}
```
Use these exact semantics:
| Public values | Meaning |
| --- | --- |
| `ConcurrencyLimit == 0`, `QueueCapacity == nil` | Unlimited backend; preserve current behavior. |
| `ConcurrencyLimit > 0`, `QueueCapacity == nil` | Limit active generations and use the default waiting capacity of 1024. |
| `ConcurrencyLimit > 0`, `QueueCapacity != nil` | Limit active generations and use the pointed-to capacity exactly, including zero. |
| `ConcurrencyLimit < 0` | Invalid engine configuration. |
| `QueueCapacity != nil` and `*QueueCapacity < 0` | Invalid engine configuration. |
| `ConcurrencyLimit == 0` and `QueueCapacity != nil` | Invalid engine configuration because a queue without an active limit has no defined consumer value. |
`ConcurrencyLimit` counts simultaneous calls to the engine-owned internal
model-client boundary for this backend. `QueueCapacity` controls additional
accepted `Run` invocations beyond that limit. The maximum admitted runs for a
limited backend is therefore:
```text
ConcurrencyLimit + effective QueueCapacity
```
Guard that addition against integer overflow during backend validation.
Do not impose an arbitrary upper bound beyond non-negativity and overflow
safety.
The `QueueCapacity` pointer exists only to distinguish omission from explicit
zero. `WithBackend` and `NewEngine` must not retain the caller's pointer.
`Backend` continues to have no stable JSON representation, and consumers
remain directed to keyed literals.
Do not add concurrency fields to `Profile`, `ExecutionTarget`,
`ExecutionTargetOverride`, `RunRequest`, prompt or profile files, or stable
prepared/result JSON.
### Built-In And Custom Defaults
The backend registry owns these exact operational defaults:
```go
const (
openRouterConcurrencyLimit = 16
defaultQueueCapacity = 1024
)
```
The built-in `openrouter` definition has a normalized concurrency limit of 16
and queue capacity of 1024.
Consumer registrations remain unlimited when concurrency is omitted. For a
consumer backend with a positive limit and omitted queue capacity, normalize
the queue capacity to 1024. Preserve an explicitly configured zero.
Consumers still cannot replace the reserved `openrouter` registration.
Endpoint-only profiles have no backend policy and remain unlimited. A selected
backend retains its pool when a profile or request overrides only its endpoint.
### Internal Backend Representation
Extend `internal/domain.Backend` with scalar policy values and explicit
presence rather than retaining a pointer:
```go
type Backend struct {
// Existing fields...
ConcurrencyLimit int
QueueCapacity int
QueueCapacitySet bool
}
type BackendCapacityPolicy struct {
ConcurrencyLimit int
QueueCapacity int
}
```
`WithBackend` converts the public pointer into `QueueCapacity` plus
`QueueCapacitySet`. Registry normalization validates the combinations above,
fills the default, and leaves every limited stored backend with
`QueueCapacitySet == true`. Unlimited stored backends retain zero values and
`QueueCapacitySet == false`.
Add this internal registry method:
```go
func (r *Registry) CapacityPolicies() map[string]domain.BackendCapacityPolicy
```
It returns a newly allocated map containing only limited backends. Values are
scalars, so callers cannot mutate registry state. The built-in OpenRouter
policy is included. `GetBackend` continues returning a defensive backend copy,
now including normalized scalar capacity metadata.
Capacity policy is operational registry metadata. Do not merge it into an
execution target or expose it to injected model clients.
### Public Capacity Error
Add this root sentinel beside the other run errors in `engine.go`:
```go
var ErrCapacityExceeded = errors.New("backend capacity exceeded")
```
Its GoDoc must state that it identifies a `Run` rejected because the selected
backend has already admitted `ConcurrencyLimit + QueueCapacity` runs. It is
not an invalid request, an LLM/provider rate-limit response, or an
`ErrLLMGenerate` failure.
The internal capacity component owns a corresponding internal
`ErrCapacityExceeded`. Add its mapping in `publicErrorFor` before the broader
generation and invalid-request cases. The public error must preserve the
internal error through wrapping while matching `ErrCapacityExceeded` with
`errors.Is`.
A capacity rejection returns no partial result and must not invoke the
artifact reader, renderer, schema loader, validator, or model client. Prompt,
profile, and backend loading needed to select the pool may already have
occurred.
### Internal Capacity Component
Add `internal/capacity` as the single owner of engine-local run admission and
active-generation permits.
Use these package-level boundaries:
```go
var ErrCapacityExceeded error
type Manager struct {
// Private immutable pool map.
}
func NewManager(
policies map[string]domain.BackendCapacityPolicy,
) (*Manager, error)
func (m *Manager) Admit(
ctx context.Context,
backendID string,
) (release func(), err error)
func NewClient(m *Manager, next llm.Client) llm.Client
```
`NewManager` copies the supplied map and creates one independent pool per
limited backend. Defensively reject blank IDs, non-positive concurrency
limits, negative queue capacities, or total-capacity overflow even though the
registry normally supplies normalized values. Construction creates no worker
goroutines.
An absent manager, blank backend ID, or ID absent from the policy map is
unlimited:
- `Admit` succeeds with a non-nil no-op release function; and
- the client wrapper calls the next client directly.
For a limited pool, `Admit` is immediate and context-aware:
1. return `ctx.Err()` if the context is already done;
2. under the pool lock, compare admitted runs with
`ConcurrencyLimit + QueueCapacity`;
3. return an error matching internal `ErrCapacityExceeded` when full; or
4. increment admitted runs and return an idempotent release function.
The release function decrements admission exactly once, even if accidentally
called more than once. It does not release an active-generation permit; those
permits have their own lifetime.
### FIFO Generation Permits
`NewClient` returns an internal `llm.Client` wrapper around either the built-in
client or the public-client adapter. It must preserve requests, successful
responses, nil responses, and collaborator error identities exactly.
`next` must be non-nil; `NewEngine` and internal runner construction maintain
that invariant. A nil manager returns `next` unchanged.
For a configured backend ID, the wrapper:
1. acquires one active-generation permit from the matching pool;
2. waits in FIFO order when the active count equals `ConcurrencyLimit`;
3. removes a canceled waiter and returns `ctx.Err()` when cancellation wins
before the permit is granted;
4. invokes the next client only after a permit is granted; and
5. releases the permit with `defer` after every success, nil response,
collaborator error, panic unwinding, or context outcome.
Implement FIFO and cancellation explicitly with a mutex and an ordered waiter
list. A channel used only as a counting semaphore is insufficient because it
does not define FIFO ordering or safe removal of canceled waiters.
Permit grant and cancellation must have one lock-protected linearization
point. If cancellation removes the waiter first, do not invoke the next
client. If grant wins first, invoke the next client with the caller's context;
the next client may then observe cancellation normally. Never lose or
double-release a permit in this race.
Releasing a permit transfers it to the oldest non-canceled waiter before
making it generally available. Different backend pools never share admission
or active counts.
The active wrapper enforces its limit even if an internal caller invokes it
without a run admission lease. Bounded backlog is guaranteed for ordinary
engine `Run` calls by the runner admission path; no public API exposes the
wrapped internal client directly.
### Engine Assembly
In `NewEngine`, after constructing the validated backend registry:
1. obtain `backendRegistry.CapacityPolicies()`;
2. construct one `capacity.Manager`;
3. construct the selected base internal LLM client exactly as today;
4. wrap that base client with `capacity.NewClient`; and
5. pass both the wrapped client and manager-as-admitter to the runner.
Every `NewEngine` call constructs a distinct manager. Do not cache managers,
pools, or policies in package globals. The wrapper must be applied after a
public injected client is adapted to `internal/llm.Client`, so built-in and
injected clients receive identical scheduling behavior.
If `NewManager` reports a defensive configuration error, make `NewEngine`
return an error matching `ErrInvalidConfig`.
`Prepare` does not use the manager. An injected client remains required to be
safe for concurrent calls because different backend pools and unlimited
backends may still invoke it concurrently.
### Shared Two-Phase Preparation
Refactor `internal/usecase.Runner` so `Prepare` and `Run` share one preparation
pipeline with two private phases. Do not duplicate prompt/profile/backend
selection or execution precedence.
The first phase resolves only the state required before admission:
1. validate `PromptID`;
2. normalize the direct session ID;
3. load the prompt definition;
4. hash the original prompt definition at its existing error-order position;
5. select and load the execution profile;
6. resolve the selected backend;
7. resolve and validate the effective execution target and credentials; and
8. resolve the effective output contract without loading its schema.
Return a private state value containing the loaded definition, normalized
direct session, prompt-definition hash, selected profile ID, effective target,
numeric-presence metadata, effective output contract, and preparation start
time. Keep this value private to `internal/usecase`.
The second phase consumes that state and performs:
1. structured-output schema loading;
2. artifact loading and input hashing;
3. message and prompt-session rendering;
4. direct-session application;
5. rendered-prompt hashing; and
6. `PreparedRun` construction and timing.
Preserve every existing precedence rule, error identity, direct-session
template bypass, hash input, selected identity, copy guarantee, and timing
field. Do not reload the prompt, profile, or backend between phases.
`Runner.Prepare` records its start time, runs both phases consecutively, and
never calls admission. Its behavior and error ordering remain unchanged.
`Runner.Run` records its existing run start time, runs the first preparation
phase, and then calls:
```go
release, err := r.admitter.Admit(ctx, effectiveBackendID)
```
Use a narrow use-case-owned interface with the same signature:
```go
type RunAdmitter interface {
Admit(context.Context, string) (func(), error)
}
```
A nil admitter means unlimited behavior for internal constructors and tests.
On successful admission, immediately `defer release()` around the remainder of
the run. Then run the second preparation phase, initial generation,
validation, and all repair attempts.
If admission returns internal `capacity.ErrCapacityExceeded`, add useful
backend context without changing its identity. If it returns `ctx.Err()`,
preserve that identity directly rather than recategorizing it as invalid
request or generation failure.
This refactor intentionally replaces the current literal `Run`-calls-`Prepare`
implementation with shared private phases. Update current-state documentation
to describe one shared pipeline rather than retaining an inaccurate call-graph
claim.
### Generation And Repair Lifetime
The admission lease covers the entire accepted run:
- second-phase preparation;
- initial generation;
- validation;
- every repair; and
- all failure and cancellation exits.
Preparation and validation do not hold an active-generation permit. The
wrapped client acquires a permit only around each actual `Generate` call.
The runner's initial generation already carries the effective backend ID in
`GenerateRequest.Target`. Preserve that value. `RepairRequest.Target` and the
default repairer's generated request must continue carrying the same backend
ID, allowing each repair to reacquire the same pool's active permit.
When testing or constructing `NewRunnerWithRepairer`, pass the same wrapped
client to both the runner and `NewDefaultOutputRepairer`. Do not add capacity
state to `RepairRequest`, `ExecutionTarget`, or public generation values.
A repair remains within its existing admission lease. It waits for a FIFO
active permit but never performs a second bounded admission and therefore
cannot fail merely because later runs filled the admission capacity.
### Error And Cancellation Semantics
The required public outcomes are:
| Situation | Required error identity |
| --- | --- |
| Admission capacity is full | `ErrCapacityExceeded` only; not `ErrInvalidRequest` or `ErrLLMGenerate`. |
| Context is done before admission succeeds | Preserve `ctx.Err()`; do not return capacity exhaustion. |
| Context cancels while waiting for an active permit | Preserve `ctx.Err()` through the existing `ErrLLMGenerate` generation category. |
| Wrapped client fails after permit acquisition | Preserve existing `ErrLLMGenerate` and collaborator identities. |
| Preparation or validation fails after admission | Preserve its existing category and release admission. |
Maintain the existing rule that `Run` returns no partial result on any
operational error. Do not add queue status to errors or results.
`RunResult.Duration` continues to start at runner entry and therefore includes
pre-admission resolution, accepted preparation, and active-permit waiting.
`PreparedRun.DurationMS` continues to cover only its shared preparation phases;
it does not include later generation waiting. Capacity-rejected calls have no
result or timing value.
### Ownership And Concurrency Safety
The registry, capacity policy map, pool map, and per-pool limits are immutable
after engine construction. Only admission counts, active counts, and waiter
lists are mutable and must be protected by the owning pool mutex.
Do not retain public queue pointers, caller request values, contexts, or
generation requests after their call completes. A canceled waiter must be
unlinked so its context and request cannot remain reachable from the pool.
Do not hold a pool mutex while:
- loading or rendering prompts;
- reading artifacts or schemas;
- invoking a model client;
- validating output;
- closing a waiter notification channel if the implementation could re-enter
pool code; or
- calling consumer code.
No scheduler operation may spawn a goroutine whose lifetime outlasts the
calling `Run`. The zero steady-state goroutine count is part of the
in-process/no-worker-lifecycle design.
## Test Ownership
Use this ownership split and avoid repeating the full policy matrix at every
layer:
- `internal/backend/registry_test.go` owns normalization, validation, the exact
OpenRouter policy, the custom default queue, explicit zero, unlimited
omission, and policy-map copying.
- `internal/capacity/manager_test.go` owns admission bounds, idempotent release,
FIFO active permits, cancellation races, capacity recovery, independent
pools, unlimited IDs, and observed peak concurrency.
- `internal/capacity/client_test.go` owns wrapper request/response/error
transparency and the rule that cancellation before grant does not invoke the
next client. Combine these with manager tests if one coherent package test
expresses the behavior more clearly.
- `internal/usecase/runner_test.go` owns two-phase preparation parity, pool
selection, admission before expensive work, admission release across run
exits, `Prepare` bypass, and repair reuse of the admitted backend.
- Root external-package tests own public configuration conversion, assembled
engine-local behavior, endpoint-override routing, injected-client limiting,
and public capacity/context error identities.
- Existing model-client HTTP tests remain unchanged because scheduling does
not alter the OpenAI-compatible wire contract.
Concurrency tests must use test-owned limits such as one or two and
channel-controlled blocking clients. Record observed active and peak counts
under a mutex or atomics. Do not use `time.Sleep` to infer queue state.
Package-internal tests may inspect a waiter list under its mutex through a
small test helper when necessary to establish deterministic FIFO ordering; do
not add production metrics or hooks solely for tests.
Do not add separate tests for trivial scalar copies when registry or assembled
behavior already protects them.
## Stage 1 — Backend Policy And Public Configuration
**Status:** Pending.
### Goal
Add the public and internal backend policy representation, normalize all
configured states, and expose immutable normalized policies without changing
runtime scheduling yet.
### Work
1. Add `ConcurrencyLimit` and `QueueCapacity` to `Backend` in `backends.go`
with exact GoDoc for unlimited, defaulted, explicit-zero, invalid, and
engine-scoped behavior.
2. Convert the public queue pointer into scalar value plus presence in
`WithBackend`; do not retain the pointer.
3. Add the internal backend policy fields and
`BackendCapacityPolicy` to `internal/domain/domain.go`.
4. Add the two registry-owned constants and configure the built-in OpenRouter
definition with 16 and 1024.
5. Extend `normalizeBackend` with the fixed validation, defaulting, explicit
zero, and overflow rules.
6. Add `Registry.CapacityPolicies`, returning only limited policies in a fresh
map.
7. Update existing backend composite literals and assertions only where the
new fields are relevant. Continue using keyed literals.
### Tests
1. Extend the exact built-in registry test with the OpenRouter limit and queue.
2. Add one coherent table covering unlimited omission, default queue,
explicit-zero queue, negative values, queue-without-limit, and total
overflow.
3. Extend the registry copy/normalization test to prove returned policy maps
cannot mutate registry state.
4. Add root coverage only if needed to prove the public pointer/presence
conversion; do not reproduce registry validation cases at the facade.
### Focused Validation
Run:
```sh
gofmt -w backends.go internal/domain/domain.go \
internal/backend/registry.go internal/backend/registry_test.go
go test . ./internal/backend
go vet . ./internal/backend
git diff --check
```
Include another touched Go test file in `gofmt` only if it actually changed.
### Completion Gate
This stage is complete when every public configuration state has one normalized
internal meaning, OpenRouter exposes exactly 16/1024, custom backends remain
unlimited by omission, and no runtime call is scheduled yet.
## Stage 2 — Engine-Local Capacity Manager
**Status:** Pending.
### Goal
Implement and prove the bounded admission mechanism and FIFO active-generation
client wrapper independently of runner orchestration.
### Work
1. Add `internal/capacity/manager.go` with the manager, immutable policy copy,
per-backend pools, internal error, immediate admission, idempotent release,
and FIFO context-aware active permits.
2. Add `internal/capacity/client.go` with the transparent `llm.Client` wrapper.
3. Use mutex-protected waiter state and an ordered list; explicitly resolve
grant-versus-cancel races.
4. Ensure unlimited and independent-pool fast paths avoid queue allocation.
5. Do not start workers, timers, cleanup goroutines, or process-global state.
### Tests
1. Add a compact constructor-validation table for blank IDs, non-positive
limits, negative queues, and total-capacity overflow.
2. With a small configured policy, prove that exactly
`limit + queueCapacity` admissions succeed, the next matches
`ErrCapacityExceeded`, and a release permits another admission.
3. Prove release is idempotent.
4. Drive more blocked client calls than the active limit and assert observed
peak concurrency never exceeds that limit.
5. Prove FIFO order with deterministic queue-entry synchronization.
6. Cancel the first and a middle waiter and prove they are removed, never call
the wrapped client, and do not block later waiters.
7. Exercise the grant/cancel race repeatedly under `go test -race`, asserting
no permit leak or double invocation.
8. Prove different backend IDs proceed independently and blank, unknown, or
nil-manager paths remain unlimited.
9. Prove request values, successful and nil responses, and collaborator errors
pass through unchanged after permit acquisition.
### Focused Validation
Run:
```sh
gofmt -w internal/capacity/manager.go \
internal/capacity/manager_test.go \
internal/capacity/client.go \
internal/capacity/client_test.go
go test ./internal/capacity
go test -race ./internal/capacity
go vet ./internal/capacity
git diff --check
```
If tests are combined into one file, omit the nonexistent file from `gofmt`.
### Completion Gate
This stage is complete when the standalone component enforces relational
admission and active limits, FIFO cancellation is race-safe, separate pools
are independent, and the wrapper is transparent apart from waiting.
## Stage 3 — Shared Preparation And Early Run Admission
**Status:** Pending.
### Goal
Refactor runner preparation into one shared two-phase pipeline and place
bounded admission after backend resolution but before schema, artifact, and
rendering work.
### Work
1. Add the private pre-admission preparation state and split the existing
`Prepare` logic according to the fixed design.
2. Make `Runner.Prepare` call both phases without an admitter.
3. Add the `RunAdmitter` interface and runner field.
4. Update `NewRunner` and `NewRunnerWithRepairer` to accept the optional
admitter; update internal call sites with nil until root assembly is wired.
5. Change `Runner.Run` to use the first phase, admit by effective backend ID,
defer the returned release, and then use the second phase.
6. Preserve all existing error precedence, target resolution, hashes,
metadata, session behavior, and timing.
7. Return capacity and context errors with the fixed identities. Do not invoke
later collaborators after rejection.
### Tests
1. Keep the existing `Run`/`Prepare` parity coverage passing to prove the
shared phases do not drift.
2. Add a fake admitter that records backend IDs and release calls.
3. Prove a backend-selected run admits with the selected ID even when the
endpoint is overridden.
4. Prove an endpoint-only run uses the unlimited/blank identity and that
`Prepare` never calls admission.
5. Reject admission and assert schema, artifact, renderer, validator, repairer,
and LLM collaborators are not invoked.
6. Prove admission is released after one successful run and representative
second-phase, generation, and validation errors. Prefer a small table around
the single `defer` invariant rather than duplicating every error test.
7. Retain direct-session, backend precedence, credential, hashing, and repair
tests unchanged except for constructor arguments.
### Focused Validation
Run:
```sh
gofmt -w internal/usecase/runner.go \
internal/usecase/runner_test.go
go test ./internal/usecase
go test -race ./internal/usecase
go vet ./internal/usecase
git diff --check
```
### Completion Gate
This stage is complete when `Prepare` remains unrestricted, `Run` admits after
one canonical routing phase and before expensive completion work, every exit
releases admission, and existing preparation semantics remain unchanged.
## Stage 4 — Engine Assembly And Public Runtime Contract
**Status:** Pending.
### Goal
Wire one manager into each engine, schedule built-in and injected clients,
expose the capacity error, and prove assembled runtime behavior.
### Work
1. Add public `ErrCapacityExceeded` and its exact GoDoc in `engine.go`.
2. Map internal capacity exhaustion in `errors.go`.
3. Construct the manager from the registry policy snapshot in `NewEngine`.
4. Wrap the selected internal client after built-in or injected-client
selection and pass the manager and wrapped client to the runner.
5. Update `Engine`, `NewEngine`, `Run`, `WithLLMClient`, and `LLMClient` GoDoc
only where concurrency, capacity, or cancellation statements change.
6. Ensure manager-construction errors match `ErrInvalidConfig`.
7. For internal repair coverage, construct the default repairer with the same
wrapped client used by its runner and confirm repair target backend identity
remains intact.
### Tests
1. Add an external-package assembled test with a small custom limit and a
blocking injected client; assert peak generation equals or remains below
the configured limit.
2. With queue capacity zero, block one accepted run before generation and
assert the next matching-backend run returns `ErrCapacityExceeded`, does not
match `ErrInvalidRequest` or `ErrLLMGenerate`, returns no result, and never
reaches expensive collaborators or the client.
3. In the same or another focused workflow, prove an endpoint override remains
in the selected backend's pool.
4. Prove two engines with the same backend ID have independent capacity.
5. Prove an unlimited custom backend and an endpoint-only profile preserve
concurrent behavior.
6. Cancel a call waiting for an active permit; assert it matches both
`context.Canceled` and `ErrLLMGenerate`, never invokes the injected client,
and leaves capacity reusable.
7. Add one internal repair workflow with concurrent runs or controlled permits
showing initial and repair generations never exceed the same backend limit
and repairs do not perform a second admission.
8. Extend the public error sentinel contract test with
`ErrCapacityExceeded`.
Avoid a second HTTP-level concurrency suite: the capacity client tests and one
assembled injected-client workflow already protect the shared wrapper used by
the built-in client.
### Focused Validation
Run:
```sh
gofmt -w engine.go errors.go backends.go \
internal/usecase/runner.go internal/usecase/runner_test.go \
engine_test.go public_contract_test.go
go test . ./internal/backend ./internal/capacity ./internal/usecase
go test -race . ./internal/capacity ./internal/usecase
go vet . ./internal/backend ./internal/capacity ./internal/usecase
git diff --check
```
Add any newly created capacity files to `gofmt` when they changed in this
stage.
### Completion Gate
This stage is complete when every engine has independent pools, limited runs
are bounded and FIFO at generation, endpoint routing is correct, capacity and
context errors are stable, repairs reuse admission, and both client kinds pass
through the same wrapper.
## Stage 5 — Durable Documentation And Final Validation
**Status:** Pending.
### Goal
Move implemented contracts into their durable owners, record compatibility
impact, and validate the complete repository.
### Work
1. Review every changed exported declaration. Ensure GoDoc is the canonical
owner of exact field types, nil/zero semantics, defaulting, error identity,
engine scope, concurrency safety, cancellation, and source compatibility.
2. Update `doc.go` so its concurrency summary acknowledges backend scheduling
while continuing to require injected collaborators to be concurrency-safe.
3. Update `docs/consumers/pkg-promptkit.md` with task-oriented examples for:
- a limited local backend;
- omitted queue capacity selecting 1024;
- explicit zero queue capacity; and
- handling `ErrCapacityExceeded`.
Keep exact field semantics in GoDoc rather than duplicating a full table.
4. Add `docs/internal/capacity.md` as the durable owner of pool lifecycle,
admission, FIFO active permits, cancellation, client wrapping, and test
ownership.
5. Add `internal/capacity` to `docs/internal/overview.md`.
6. Update `docs/policy/architecture.md` to include the implemented component
and root assembly dependency without turning policy into an API reference.
7. Update `docs/internal/runner.md` to describe the shared two-phase
preparation pipeline, early bounded admission, lease lifetime, generation
permits, repairs, capacity failures, and cancellation.
8. Review `docs/formats.md`; add only a concise link or clarification if needed
to explain that endpoint overrides preserve backend capacity identity.
Do not add concurrency fields to YAML.
9. Do not change the OpenAI-compatible integration contract or
`docs/internal/llm.md` unless implementation changes their current
statements; scheduling is outside the provider wire contract and concrete
model-client implementation.
10. Record in the implementation handoff that built-in OpenRouter now limits
active generations to 16 with queue capacity 1024 and that the release
must be a pre-`v1` minor release. Do not edit the release procedure or
create a tag.
11. After every check passes, set `concurrency.md`, this implementation plan,
and each stage status to `Complete`. Do not remove the roadmaps in the
implementation change; lifecycle retirement follows review.
### Full Validation
Run the complete sequence from `docs/development.md`:
```sh
go test ./...
go test -race ./...
go vet ./...
go build ./...
go run ./examples/go-library/prepare
gofmt -l $(git ls-files '*.go')
git diff --check
```
The formatting command must produce no paths. Follow every added or changed
Markdown link and confirm its target and heading exist.
Also inspect:
```sh
git status --short
git diff --stat
git diff
```
Confirm that:
- only intended backend, capacity, runner, facade, test, documentation, and
roadmap files changed;
- no `go.work`, `go.work.sum`, local module replacement, credential,
generated binary, coverage output, or unrelated change was introduced;
- the built-in OpenRouter policy is exactly 16/1024;
- custom and endpoint-only backends remain unlimited by omission;
- explicit queue zero is distinguishable from omission;
- no capacity value enters execution targets, generated requests, stable JSON,
prompt/profile YAML, or provider payloads;
- every engine owns distinct pools with no package-global mutable state;
- every initial and repair generation uses the active permit wrapper;
- capacity and waiter state is released on success, error, panic unwinding,
and cancellation;
- concurrency tests use deterministic coordination rather than sleeps;
- current-state documentation describes implemented behavior rather than
referring readers to the roadmaps; and
- the feature and implementation roadmaps contain no unresolved work marked
complete.
### Completion Gate
The implementation is complete only when every target-end-state item in
`concurrency.md` is implemented, race-enabled tests demonstrate the configured
limits and cancellation safety, durable contracts no longer depend on roadmap
prose, and the OpenRouter compatibility change is clearly reported for the
next minor release.
## Open Questions
None. The feature roadmap and this plan fix the public representation,
registry defaults, admission bound, FIFO generation behavior, early-routing
refactor, cancellation races, error identities, engine and repair lifetimes,
test ownership, compatibility treatment, and non-goals required for
implementation.

View File

@@ -12,7 +12,9 @@ import (
"time"
artifactadapter "gitea.maximumdirect.net/eric/promptkit/internal/artifact"
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
"gitea.maximumdirect.net/eric/promptkit/internal/profile/builtin"
@@ -23,7 +25,8 @@ import (
)
// ErrInvalidConfig identifies invalid engine construction, including missing
// required configuration, invalid options, and a nil Engine receiver.
// required configuration, invalid options or backend registrations, and a nil
// Engine receiver.
var ErrInvalidConfig = errors.New("invalid engine configuration")
var (
@@ -46,8 +49,8 @@ var (
// ErrPromptNotFound.
ErrPromptLoad = errors.New("failed to load prompt definition")
// ErrProfileLoad identifies a failure to read, decode, validate, or select
// an execution profile, except for the not-found case represented by
// ErrProfileNotFound.
// an execution profile or resolve its backend, except for the profile
// not-found case represented by ErrProfileNotFound.
ErrProfileLoad = errors.New("failed to load execution profile")
// ErrAPIKeyEnvMissing identifies an APIKeyEnv whose environment variable is
// unset or empty when no direct RunRequest.APIKey takes precedence. Such an
@@ -107,8 +110,10 @@ type Config struct {
// NewEngine applies options in argument order and ignores nil options. Within
// each prompt-source, profile-source, in-memory-profile, schema-source,
// model-client, and artifact-reader category, the last non-nil valid option
// replaces earlier options in that category. An invalid option fails
// construction even if a later option would replace it.
// replaces earlier options in that category. WithBackend is the additive
// exception: unique registrations accumulate, and a repeated backend ID is an
// error rather than a replacement. An invalid option fails construction even
// if a later option would replace it.
type Option interface {
apply(*engineOptions) error
}
@@ -125,6 +130,7 @@ type engineOptions struct {
promptDefs promptdef.Repository
profiles profile.Repository
memoryProfiles profile.Repository
backends []domain.Backend
validator validate.Validator
promptSource bool
profileSource bool
@@ -302,8 +308,9 @@ func WithSchemaFile(path string) Option {
//
// Options are applied in order according to [Option]. PromptDir is required
// unless a prompt-source option is present. Construction validates option
// arguments and in-memory profiles but defers reading and validating prompt,
// file-backed profile, and schema contents until Prepare or Run needs them.
// arguments, in-memory profiles, and backend registrations but defers reading
// and validating prompt, file-backed profile, and schema contents until Prepare
// or Run needs them.
//
// NewEngine returns an error matching ErrInvalidConfig for invalid
// configuration or options. It does not perform model requests or require
@@ -335,6 +342,11 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
profiles = profile.NewOverlayRepository(options.memoryProfiles, profiles)
}
backendRegistry, err := backend.NewRegistry(options.backends)
if err != nil {
return nil, fmt.Errorf("%w: failed to construct backend registry: %v", ErrInvalidConfig, err)
}
validator := options.validator
if !options.validatorSource {
schemaDir := cfg.SchemaDir
@@ -365,6 +377,7 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
runner: usecase.NewRunner(
promptDefs,
profiles,
backendRegistry,
artifacts,
prompt.NewGoRenderer(),
llmClient,
@@ -395,11 +408,12 @@ func fileSource(name string) (fs.FS, string, error) {
// Prepare resolves and renders a prompt request without calling an LLM.
//
// Prepare selects the prompt and profile, resolves effective execution
// settings and the output contract, loads and hashes inputs, loads structured
// output schema metadata when required, and renders the session ID and
// messages. The returned PreparedRun is owned by the caller and never contains
// a resolved API-key value, model output, or validation result.
// Prepare selects the prompt and profile, resolves any selected backend and
// effective execution settings, resolves the output contract, loads and hashes
// inputs, loads structured-output schema metadata when required, and renders
// the session ID and messages. The returned PreparedRun is owned by the caller
// and never contains a resolved API-key value, model output, or validation
// result.
//
// A nil Engine returns an error matching ErrInvalidConfig. Request and
// preparation failures may match ErrInvalidRequest, ErrPromptNotFound,

View File

@@ -281,6 +281,9 @@ func TestEngineExecutionSettingPrecedence(t *testing.T) {
intPointer := func(value int) *int {
return &value
}
stringPointer := func(value string) *string {
return &value
}
defaultsProfile := executionProfileFixture{
id: "settings-defaults",
@@ -388,7 +391,7 @@ func TestEngineExecutionSettingPrecedence(t *testing.T) {
TopP: floatPointer(requestTarget.TopP),
TimeoutSeconds: intPointer(requestTarget.TimeoutSeconds),
ServiceTier: requestTarget.ServiceTier,
ReasoningEffort: requestTarget.ReasoningEffort,
ReasoningEffort: stringPointer(requestTarget.ReasoningEffort),
APIKeyEnv: requestTarget.APIKeyEnv,
ExtraParams: requestTarget.ExtraParams,
},
@@ -452,6 +455,43 @@ func TestEngineExecutionSettingPrecedence(t *testing.T) {
}
})
}
t.Run("blank request reasoning clears profile setting", func(t *testing.T) {
profile := executionProfileFixture{
id: "settings-reasoning-clear",
endpoint: "http://profile-reasoning.test/v1",
model: "profile-reasoning-model",
reasoningEffort: "medium",
}
profileDir := t.TempDir()
writeExecutionProfileFixture(t, profileDir, profile)
engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: frameworkPromptDir,
ProfileDir: profileDir,
SchemaDir: frameworkSchemaDir,
})
if err != nil {
t.Fatalf("construct engine: %v", err)
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: frameworkMarkdownSummaryPromptID,
ProfileID: profile.id,
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline("Nia labels the archive."),
"glossary": promptkit.Inline("archive: A catalogued collection."),
},
Execution: &promptkit.ExecutionTargetOverride{
ReasoningEffort: stringPointer(" \t "),
},
})
if err != nil {
t.Fatalf("prepare engine: %v", err)
}
if prepared.EffectiveModelParams.ReasoningEffort != "" {
t.Fatalf("expected blank request reasoning to clear profile value, got %q", prepared.EffectiveModelParams.ReasoningEffort)
}
})
}
func TestRunSucceedsWithInjectedLLMClient(t *testing.T) {
@@ -571,19 +611,26 @@ func TestEngineRunWithDirectorySourcesAndFileInputs(t *testing.T) {
func TestRunPassesPreparedRequestToInjectedLLMClient(t *testing.T) {
const directKey = "direct-injected-key"
const directSession = "assembled-session"
fake := &fakeLLMClient{
response: &promptkit.GenerateResponse{Content: `{"events":[{"title":"Archive labelled"}]}`},
}
engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(fake))
_, err := engine.Run(context.Background(), promptkit.RunRequest{
PromptID: frameworkStructuredEventsPromptID,
APIKey: directKey,
runRequest := promptkit.RunRequest{
PromptID: frameworkStructuredEventsPromptID,
SessionID: " " + directSession + " ",
APIKey: directKey,
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline("Rin opens the gate."),
"glossary": promptkit.Inline("gate: A guarded passage."),
},
})
}
prepared, err := engine.Prepare(context.Background(), runRequest)
if err != nil {
t.Fatalf("expected prepare to succeed, got %v", err)
}
result, err := engine.Run(context.Background(), runRequest)
if err != nil {
t.Fatalf("expected run to succeed, got %v", err)
}
@@ -594,6 +641,16 @@ func TestRunPassesPreparedRequestToInjectedLLMClient(t *testing.T) {
if len(req.Prompt.Messages) != 2 || !strings.Contains(req.Prompt.Messages[1].Content, "Rin opens the gate.") {
t.Fatalf("expected rendered prompt in generate request, got %+v", req.Prompt)
}
if prepared.SessionID != directSession ||
req.Prompt.SessionID != directSession ||
result.SessionID != directSession {
t.Fatalf(
"direct session did not propagate consistently: prepared=%q generated=%q result=%q",
prepared.SessionID,
req.Prompt.SessionID,
result.SessionID,
)
}
if req.StructuredOutput == nil || req.StructuredOutput.Type != promptkit.StructuredOutputJSONSchema || req.StructuredOutput.JSONSchema == nil {
t.Fatalf("expected structured output handoff, got %+v", req.StructuredOutput)
}
@@ -755,6 +812,92 @@ func TestRunUsesDirectAPIKeyWithDefaultLLMClient(t *testing.T) {
}
}
func TestRunUsesResolvedBackendWithBuiltInLLMClient(t *testing.T) {
const (
backendID = "local-test"
envName = "PROMPTKIT_BACKEND_TRANSPORT_KEY"
apiKey = "synthetic-backend-key"
)
t.Setenv(envName, apiKey)
var (
gotAuth string
gotBody map[string]any
)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
gotAuth = r.Header.Get("Authorization")
if r.URL.Path != "/v1/chat/completions" {
t.Errorf("unexpected path: %s", r.URL.Path)
}
if err := json.NewDecoder(r.Body).Decode(&gotBody); err != nil {
t.Errorf("decode request body: %v", err)
}
w.Header().Set("Content-Type", "application/json")
_, _ = w.Write([]byte(`{
"choices": [{"message": {"role": "assistant", "content": "# Summary\n\nDone."}}],
"usage": {"prompt_tokens": 3, "completion_tokens": 4, "total_tokens": 7}
}`))
}))
defer server.Close()
engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: frameworkPromptDir,
SchemaDir: frameworkSchemaDir,
},
promptkit.WithBackend(promptkit.Backend{
ID: backendID,
Endpoint: server.URL + "/v1",
APIKeyEnv: envName,
ExtraParams: map[string]any{
"provider": "synthetic",
},
}),
promptkit.WithProfiles(promptkit.Profile{
ID: "backend-transport",
BackendID: backendID,
Model: "test-model",
}),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
result, err := engine.Run(context.Background(), promptkit.RunRequest{
PromptID: frameworkMarkdownSummaryPromptID,
ProfileID: "backend-transport",
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline("Rin opens the gate."),
"glossary": promptkit.Inline("gate: A guarded passage."),
},
})
if err != nil {
t.Fatalf("run with resolved backend: %v", err)
}
if gotAuth != "Bearer "+apiKey {
t.Fatalf("unexpected Authorization header: %q", gotAuth)
}
if gotBody["model"] != "test-model" || gotBody["provider"] != "synthetic" {
t.Fatalf("backend defaults did not reach provider payload: %#v", gotBody)
}
for _, field := range []string{"backend_id", "api_key_env"} {
if _, ok := gotBody[field]; ok {
t.Fatalf("internal metadata field %q was serialized to provider payload: %#v", field, gotBody)
}
}
bodyJSON, err := json.Marshal(gotBody)
if err != nil {
t.Fatalf("marshal captured provider payload: %v", err)
}
if strings.Contains(string(bodyJSON), apiKey) {
t.Fatalf("credential value was serialized to provider payload: %s", bodyJSON)
}
if result.SelectedBackendID != backendID ||
result.EffectiveModelParams.Endpoint != server.URL+"/v1" ||
result.EffectiveModelParams.APIKeyEnv != envName {
t.Fatalf("unexpected resolved backend metadata: %+v", result)
}
}
func TestPrepareDirectAPIKeyBypassesMissingEnvWithoutLeakingOrHashing(t *testing.T) {
const missingEnv = "PROMPTKIT_PUBLIC_PREPARE_MISSING"
const firstKey = "first-direct-key"
@@ -1271,6 +1414,18 @@ func TestPrepareUsesBuiltInProfileWithoutProfileDir(t *testing.T) {
if prepared.SelectedProfileID != "mistral-small-3" {
t.Fatalf("unexpected selected profile: %q", prepared.SelectedProfileID)
}
if prepared.SelectedBackendID != promptkit.BackendOpenRouter {
t.Fatalf("unexpected selected backend: %q", prepared.SelectedBackendID)
}
if prepared.EffectiveModelParams.BackendID != promptkit.BackendOpenRouter {
t.Fatalf("unexpected effective backend: %q", prepared.EffectiveModelParams.BackendID)
}
if prepared.EffectiveModelParams.Endpoint != "https://openrouter.ai/api/v1" {
t.Fatalf("unexpected built-in endpoint: %q", prepared.EffectiveModelParams.Endpoint)
}
if prepared.EffectiveModelParams.APIKeyEnv != "OPENROUTER_API_KEY" {
t.Fatalf("unexpected built-in api key environment name: %q", prepared.EffectiveModelParams.APIKeyEnv)
}
if prepared.EffectiveModelParams.Model != "mistralai/mistral-small-3.2-24b-instruct" {
t.Fatalf("unexpected built-in model: %q", prepared.EffectiveModelParams.Model)
}
@@ -1641,6 +1796,7 @@ func TestOpenAICompatibleProfileRunsThroughNormalProfilePath(t *testing.T) {
fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
prof := promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "template-profile",
BackendID: " openrouter ",
Endpoint: "http://template/v1",
Model: "template-model",
APIKeyRequired: true,
@@ -1672,7 +1828,9 @@ func TestOpenAICompatibleProfileRunsThroughNormalProfilePath(t *testing.T) {
if len(fake.requests) != 1 {
t.Fatalf("expected one request, got %d", len(fake.requests))
}
if fake.requests[0].Target.Model != "template-model" || fake.requests[0].APIKey != "template-key" {
if fake.requests[0].Target.BackendID != promptkit.BackendOpenRouter ||
fake.requests[0].Target.Model != "template-model" ||
fake.requests[0].APIKey != "template-key" {
t.Fatalf("unexpected generated request: %+v", fake.requests[0])
}
if !reflect.DeepEqual(fake.requests[0].Target.ExtraParams, map[string]any{"provider": "template"}) {
@@ -2256,6 +2414,11 @@ func TestExtraParamsTypedNestedValuesAreCopiedAcrossPublicBoundary(t *testing.T)
}
func TestRunRejectsInvalidExtraParams(t *testing.T) {
cyclicMap := map[string]any{}
cyclicMap["self"] = cyclicMap
cyclicSlice := []any{nil}
cyclicSlice[0] = cyclicSlice
tests := []struct {
name string
extraParams map[string]any
@@ -2267,42 +2430,10 @@ func TestRunRejectsInvalidExtraParams(t *testing.T) {
{name: "nan", extraParams: map[string]any{"bad": math.NaN()}},
{name: "positive infinity", extraParams: map[string]any{"bad": math.Inf(1)}},
{name: "negative infinity", extraParams: map[string]any{"bad": math.Inf(-1)}},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(fake))
_, err := engine.Run(context.Background(), promptkit.RunRequest{
PromptID: frameworkMarkdownSummaryPromptID,
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline("Rin opens the gate."),
"glossary": promptkit.Inline("gate: A guarded passage."),
},
Execution: &promptkit.ExecutionTargetOverride{ExtraParams: tc.extraParams},
})
if !errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("expected ErrInvalidRequest, got %v", err)
}
if len(fake.requests) != 0 {
t.Fatalf("expected invalid request to fail before LLM call, got %d requests", len(fake.requests))
}
})
}
}
func TestRunRejectsCyclicExtraParams(t *testing.T) {
cyclicMap := map[string]any{}
cyclicMap["self"] = cyclicMap
cyclicSlice := []any{nil}
cyclicSlice[0] = cyclicSlice
tests := []struct {
name string
extraParams map[string]any
}{
{name: "map", extraParams: cyclicMap},
{name: "slice", extraParams: map[string]any{"cycle": cyclicSlice}},
{name: "cyclic map", extraParams: cyclicMap},
{name: "cyclic slice", extraParams: map[string]any{"cycle": cyclicSlice}},
{name: "malformed JSON number", extraParams: map[string]any{"value": json.Number("+1")}},
{name: "empty nested key", extraParams: map[string]any{"nested": map[string]any{"": true}}},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {

View File

@@ -0,0 +1,155 @@
// Package backend owns validated, immutable OpenAI-compatible backend
// definitions.
package backend
import (
"errors"
"fmt"
"net/url"
"regexp"
"sort"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
)
const (
// OpenRouterID is the reserved ID of Promptkit's built-in OpenRouter
// backend.
OpenRouterID = "openrouter"
openRouterEndpoint = "https://openrouter.ai/api/v1"
openRouterAPIKeyEnv = "OPENROUTER_API_KEY"
)
// ErrBackendNotFound identifies a registry lookup for an unknown backend ID.
var ErrBackendNotFound = errors.New("backend not found")
var environmentVariableName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
// Registry is an immutable collection of validated backend definitions.
type Registry struct {
backends map[string]domain.Backend
}
// NewRegistry constructs a registry containing the built-in OpenRouter
// definition followed by the supplied additions. Every ID must be unique.
func NewRegistry(additions []domain.Backend) (*Registry, error) {
registry := &Registry{
backends: make(map[string]domain.Backend, len(additions)+1),
}
definitions := make([]domain.Backend, 0, len(additions)+1)
definitions = append(definitions, domain.Backend{
ID: OpenRouterID,
Endpoint: openRouterEndpoint,
APIKeyEnv: openRouterAPIKeyEnv,
})
definitions = append(definitions, additions...)
for _, definition := range definitions {
definition.ID = strings.TrimSpace(definition.ID)
if definition.ID == "" {
return nil, errors.New("backend ID must not be blank")
}
if _, exists := registry.backends[definition.ID]; exists {
return nil, fmt.Errorf("backend ID %q is already registered", definition.ID)
}
normalized, err := normalizeBackend(definition)
if err != nil {
return nil, err
}
registry.backends[normalized.ID] = normalized
}
return registry, nil
}
// GetBackend returns a defensive copy of the backend registered with id.
func (r *Registry) GetBackend(id string) (domain.Backend, error) {
if r == nil {
return domain.Backend{}, fmt.Errorf("%w: %q", ErrBackendNotFound, id)
}
definition, ok := r.backends[id]
if !ok {
return domain.Backend{}, fmt.Errorf("%w: %q", ErrBackendNotFound, id)
}
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
if err != nil {
return domain.Backend{}, fmt.Errorf("copy backend %q: %w", id, err)
}
definition.ExtraParams = extraParams
return definition, nil
}
func normalizeBackend(definition domain.Backend) (domain.Backend, error) {
definition.Endpoint = strings.TrimSpace(definition.Endpoint)
if err := validateEndpoint(definition.Endpoint); err != nil {
return domain.Backend{}, fmt.Errorf("backend %q endpoint: %w", definition.ID, err)
}
definition.APIKeyEnv = strings.TrimSpace(definition.APIKeyEnv)
if definition.APIKeyEnv != "" && !environmentVariableName.MatchString(definition.APIKeyEnv) {
return domain.Backend{}, fmt.Errorf(
"backend %q api key environment variable %q is invalid",
definition.ID,
definition.APIKeyEnv,
)
}
keys := make([]string, 0, len(definition.ExtraParams))
for key := range definition.ExtraParams {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
if key == "" {
return domain.Backend{}, fmt.Errorf("backend %q extra parameter key must not be empty", definition.ID)
}
if llm.IsReservedOpenAIChatRequestField(key) {
return domain.Backend{}, fmt.Errorf(
"backend %q extra parameter %q collides with a reserved request field",
definition.ID,
key,
)
}
}
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
if err != nil {
return domain.Backend{}, fmt.Errorf("backend %q extra parameters: %w", definition.ID, err)
}
definition.ExtraParams = extraParams
return definition, nil
}
func validateEndpoint(endpoint string) error {
if endpoint == "" {
return errors.New("must not be blank")
}
if strings.Contains(endpoint, "#") {
return errors.New("must not contain a fragment")
}
parsed, err := url.Parse(endpoint)
if err != nil {
return fmt.Errorf("must be a valid URL: %w", err)
}
scheme := strings.ToLower(parsed.Scheme)
if scheme != "http" && scheme != "https" {
return errors.New("must use http or https")
}
if !parsed.IsAbs() || parsed.Hostname() == "" {
return errors.New("must be absolute and include a host")
}
if parsed.User != nil {
return errors.New("must not contain user information")
}
if parsed.RawQuery != "" || parsed.ForceQuery {
return errors.New("must not contain a query string")
}
return nil
}

View File

@@ -0,0 +1,225 @@
package backend_test
import (
"errors"
"strings"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
const validEndpoint = "https://backend.example/v1"
func TestRegistryIncludesExactOpenRouterDefinition(t *testing.T) {
registry, err := backend.NewRegistry(nil)
if err != nil {
t.Fatalf("construct registry: %v", err)
}
definition, err := registry.GetBackend(backend.OpenRouterID)
if err != nil {
t.Fatalf("look up OpenRouter: %v", err)
}
if definition.ID != "openrouter" ||
definition.Endpoint != "https://openrouter.ai/api/v1" ||
definition.APIKeyEnv != "OPENROUTER_API_KEY" ||
definition.ExtraParams != nil {
t.Fatalf("unexpected OpenRouter definition: %#v", definition)
}
}
func TestRegistryNormalizesUniqueAdditionsAndIsolatesMutations(t *testing.T) {
nested := map[string]int{"limit": 2}
extraParams := map[string]any{
"count": int64(7),
"nested": nested,
}
registry, err := backend.NewRegistry([]domain.Backend{
{
ID: " custom ",
Endpoint: " https://custom.example/openai/v1 ",
APIKeyEnv: " CUSTOM_API_KEY ",
ExtraParams: extraParams,
},
{
ID: "Custom",
Endpoint: validEndpoint,
},
})
if err != nil {
t.Fatalf("construct registry: %v", err)
}
nested["limit"] = 99
extraParams["added"] = true
got, err := registry.GetBackend("custom")
if err != nil {
t.Fatalf("look up custom backend: %v", err)
}
if got.ID != "custom" ||
got.Endpoint != "https://custom.example/openai/v1" ||
got.APIKeyEnv != "CUSTOM_API_KEY" {
t.Fatalf("unexpected normalized definition: %#v", got)
}
if count, ok := got.ExtraParams["count"].(int64); !ok || count != 7 {
t.Fatalf("integer type or value changed: %#v", got.ExtraParams["count"])
}
gotNested, ok := got.ExtraParams["nested"].(map[string]int)
if !ok || gotNested["limit"] != 2 {
t.Fatalf("container type or value changed: %#v", got.ExtraParams["nested"])
}
if _, exists := got.ExtraParams["added"]; exists {
t.Fatalf("registry retained caller map: %#v", got.ExtraParams)
}
gotNested["limit"] = 100
got.ExtraParams["added"] = true
again, err := registry.GetBackend("custom")
if err != nil {
t.Fatalf("look up custom backend again: %v", err)
}
if again.ExtraParams["nested"].(map[string]int)["limit"] != 2 {
t.Fatalf("lookup exposed registry nested map: %#v", again.ExtraParams)
}
if _, exists := again.ExtraParams["added"]; exists {
t.Fatalf("lookup exposed registry map: %#v", again.ExtraParams)
}
if _, err := registry.GetBackend("Custom"); err != nil {
t.Fatalf("backend IDs should be case-sensitive: %v", err)
}
}
func TestNewRegistryRejectsDuplicateIDs(t *testing.T) {
tests := []struct {
name string
additions []domain.Backend
wantID string
}{
{
name: "built-in collision after normalization",
additions: []domain.Backend{{
ID: " openrouter ",
}},
wantID: "openrouter",
},
{
name: "consumer collision after normalization",
additions: []domain.Backend{
{ID: "custom", Endpoint: validEndpoint},
{ID: " custom ", Endpoint: "https://other.example/v1"},
},
wantID: "custom",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, err := backend.NewRegistry(tc.additions)
if err == nil {
t.Fatal("expected duplicate ID error")
}
if !strings.Contains(err.Error(), tc.wantID) {
t.Fatalf("expected error to identify %q, got %v", tc.wantID, err)
}
})
}
}
func TestNewRegistryValidatesIDs(t *testing.T) {
for _, id := range []string{"", " \t\n "} {
t.Run(id, func(t *testing.T) {
_, err := backend.NewRegistry([]domain.Backend{{
ID: id,
Endpoint: validEndpoint,
}})
if err == nil {
t.Fatal("expected blank ID error")
}
})
}
}
func TestNewRegistryValidatesEndpoints(t *testing.T) {
tests := []struct {
name string
endpoint string
}{
{name: "blank", endpoint: ""},
{name: "relative", endpoint: "/v1"},
{name: "missing host", endpoint: "https:///v1"},
{name: "unsupported scheme", endpoint: "ftp://backend.example/v1"},
{name: "user information", endpoint: "https://user@backend.example/v1"},
{name: "query", endpoint: "https://backend.example/v1?mode=chat"},
{name: "empty query", endpoint: "https://backend.example/v1?"},
{name: "fragment", endpoint: "https://backend.example/v1#chat"},
{name: "empty fragment", endpoint: "https://backend.example/v1#"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, err := backend.NewRegistry([]domain.Backend{{
ID: "custom",
Endpoint: tc.endpoint,
}})
if err == nil {
t.Fatal("expected invalid endpoint error")
}
})
}
}
func TestNewRegistryValidatesEnvironmentVariableNames(t *testing.T) {
for _, name := range []string{"1API_KEY", "API-KEY", "API KEY", "ÅPI_KEY"} {
t.Run(name, func(t *testing.T) {
_, err := backend.NewRegistry([]domain.Backend{{
ID: "custom",
Endpoint: validEndpoint,
APIKeyEnv: name,
}})
if err == nil {
t.Fatal("expected invalid environment-variable name error")
}
})
}
}
func TestNewRegistryRejectsInvalidAndReservedExtraParameters(t *testing.T) {
tests := []struct {
name string
extraParams map[string]any
}{
{name: "unsupported value", extraParams: map[string]any{"value": make(chan int)}},
{name: "reserved key", extraParams: map[string]any{"model": "override"}},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, err := backend.NewRegistry([]domain.Backend{{
ID: "custom",
Endpoint: validEndpoint,
ExtraParams: tc.extraParams,
}})
if err == nil {
t.Fatal("expected invalid extra parameters error")
}
})
}
}
func TestRegistryLookupReportsNotFound(t *testing.T) {
registry, err := backend.NewRegistry(nil)
if err != nil {
t.Fatalf("construct registry: %v", err)
}
_, err = registry.GetBackend("missing")
if !errors.Is(err, backend.ErrBackendNotFound) {
t.Fatalf("expected ErrBackendNotFound, got %v", err)
}
if !strings.Contains(err.Error(), "missing") {
t.Fatalf("expected error to identify backend, got %v", err)
}
}

View File

@@ -63,6 +63,7 @@ type RunRequest struct {
PromptID string
PromptVersion string
ProfileID string
SessionID string
APIKey string `json:"-" yaml:"-"`
Inputs map[string]ArtifactRef
Vars map[string]string
@@ -79,8 +80,10 @@ type RunResult struct {
PromptID string
PromptVersion string
PromptHash string
SessionID string
RenderedPromptHash string
SelectedProfileID string
SelectedBackendID string
ModelName string
Endpoint string
EffectiveModelParams ExecutionTarget
@@ -98,6 +101,7 @@ type PreparedRun struct {
PromptVersion string `json:"prompt_version,omitempty"`
PromptHash string `json:"prompt_hash,omitempty"`
SelectedProfileID string `json:"selected_profile_id"`
SelectedBackendID string `json:"selected_backend_id,omitempty"`
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
TargetPresence ExecutionTargetPresence `json:"-"`
OutputContract OutputContract `json:"output_contract"`
@@ -157,9 +161,18 @@ type PromptMessageTemplate struct {
CacheControl *CacheControl `yaml:"cache_control,omitempty" json:"cache_control,omitempty"`
}
// Backend describes reusable OpenAI-compatible connection defaults.
type Backend struct {
ID string
Endpoint string
APIKeyEnv string
ExtraParams map[string]any
}
// ExecutionProfile describes how and where to execute a model.
type ExecutionProfile struct {
ID string `yaml:"id"`
BackendID string `yaml:"backend"`
Endpoint string `yaml:"endpoint"`
Model string `yaml:"model"`
Temperature float64 `yaml:"temperature"`
@@ -182,7 +195,7 @@ type ExecutionTargetOverride struct {
TopP *float64 `json:"top_p,omitempty"`
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
ServiceTier string `json:"service_tier,omitempty"`
ReasoningEffort string `json:"reasoning_effort,omitempty"`
ReasoningEffort *string `json:"reasoning_effort,omitempty"`
APIKeyEnv string `json:"api_key_env,omitempty"`
ExtraParams map[string]any `json:"extra_params,omitempty"`
}
@@ -198,6 +211,7 @@ type ExecutionTargetPresence struct {
// ExecutionTarget represents effective model runtime settings for a run.
type ExecutionTarget struct {
BackendID string `yaml:"backend" json:"backend_id,omitempty"`
Endpoint string `yaml:"endpoint" json:"endpoint"`
Model string `yaml:"model" json:"model"`
Temperature float64 `yaml:"temperature" json:"temperature"`

View File

@@ -0,0 +1,19 @@
package domain
import (
"fmt"
"strings"
"unicode/utf8"
)
// NormalizeSessionID applies the shared session identifier rule.
func NormalizeSessionID(raw string) (string, error) {
normalized := strings.TrimSpace(raw)
if normalized == "" {
return "", nil
}
if length := utf8.RuneCountInString(normalized); length > SessionIDMaxLength {
return "", fmt.Errorf("session_id length %d exceeds maximum %d", length, SessionIDMaxLength)
}
return normalized, nil
}

View File

@@ -0,0 +1,57 @@
package domain
import (
"strings"
"testing"
)
func TestNormalizeSessionID(t *testing.T) {
tests := []struct {
name string
raw string
want string
wantErr bool
}{
{
name: "trims surrounding Unicode whitespace",
raw: "\u2003 session-123 \u2003",
want: "session-123",
},
{
name: "blank input is omitted",
raw: " \t\u2003 ",
want: "",
},
{
name: "maximum Unicode length is accepted",
raw: strings.Repeat("界", SessionIDMaxLength),
want: strings.Repeat("界", SessionIDMaxLength),
},
{
name: "one Unicode code point over maximum is rejected",
raw: strings.Repeat("界", SessionIDMaxLength+1),
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := NormalizeSessionID(tt.raw)
if tt.wantErr {
if err == nil {
t.Fatal("expected normalization error")
}
if !strings.Contains(err.Error(), "exceeds maximum") {
t.Fatalf("expected useful length diagnostic, got %v", err)
}
return
}
if err != nil {
t.Fatalf("normalize session id: %v", err)
}
if got != tt.want {
t.Fatalf("normalized session id = %q, want %q", got, tt.want)
}
})
}
}

View File

@@ -1,25 +1,30 @@
package promptkit
// Package jsonvalue validates and defensively copies JSON-compatible value
// trees used by public configuration and request boundaries.
package jsonvalue
import (
"encoding/json"
"fmt"
"math"
"reflect"
"sort"
"strconv"
)
const maxSafeJSONInteger = 1<<53 - 1
type jsonVisit struct {
type visit struct {
typ reflect.Type
ptr uintptr
}
func copyPublicJSONMap(src map[string]any) (map[string]any, error) {
// CopyMap validates and deeply copies an extra-parameter map while preserving
// compatible concrete map, slice, array, scalar, and number types.
func CopyMap(src map[string]any) (map[string]any, error) {
if src == nil {
return nil, nil
}
copied, err := copyPublicJSONValue(reflect.ValueOf(src), "extra_params", make(map[jsonVisit]struct{}))
copied, err := copyValue(reflect.ValueOf(src), "extra_params", make(map[visit]struct{}))
if err != nil {
return nil, err
}
@@ -30,7 +35,7 @@ func copyPublicJSONMap(src map[string]any) (map[string]any, error) {
return out, nil
}
func copyPublicJSONValue(value reflect.Value, path string, seen map[jsonVisit]struct{}) (any, error) {
func copyValue(value reflect.Value, path string, seen map[visit]struct{}) (any, error) {
if !value.IsValid() {
return nil, nil
}
@@ -38,12 +43,15 @@ func copyPublicJSONValue(value reflect.Value, path string, seen map[jsonVisit]st
if value.IsNil() {
return nil, nil
}
return copyPublicJSONValue(value.Elem(), path, seen)
return copyValue(value.Elem(), path, seen)
}
if !value.CanInterface() {
return nil, fmt.Errorf("%s: value cannot be copied", path)
}
if number, ok := value.Interface().(json.Number); ok {
if _, err := json.Marshal(number); err != nil {
return nil, fmt.Errorf("%s: invalid JSON number", path)
}
f, err := strconv.ParseFloat(number.String(), 64)
if err != nil || math.IsNaN(f) || math.IsInf(f, 0) {
return nil, fmt.Errorf("%s: invalid JSON number", path)
@@ -65,8 +73,8 @@ func copyPublicJSONValue(value reflect.Value, path string, seen map[jsonVisit]st
}
return value.Interface(), nil
case reflect.Float32, reflect.Float64:
f := value.Convert(reflect.TypeOf(float64(0))).Float()
if math.IsNaN(f) || math.IsInf(f, 0) {
number := value.Convert(reflect.TypeOf(float64(0))).Float()
if math.IsNaN(number) || math.IsInf(number, 0) {
return nil, fmt.Errorf("%s: floating-point value must be finite", path)
}
return value.Interface(), nil
@@ -74,28 +82,28 @@ func copyPublicJSONValue(value reflect.Value, path string, seen map[jsonVisit]st
if value.IsNil() {
return nil, nil
}
visit := jsonVisit{typ: value.Type(), ptr: value.Pointer()}
if _, ok := seen[visit]; ok {
current := visit{typ: value.Type(), ptr: value.Pointer()}
if _, ok := seen[current]; ok {
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
}
seen[visit] = struct{}{}
defer delete(seen, visit)
return copyPublicJSONValue(value.Elem(), path, seen)
seen[current] = struct{}{}
defer delete(seen, current)
return copyValue(value.Elem(), path, seen)
case reflect.Map:
return copyPublicJSONMapValue(value, path, seen)
return copyMapValue(value, path, seen)
case reflect.Slice:
if value.IsNil() {
return nil, nil
}
return copyPublicJSONSequenceValue(value, path, seen)
return copySequenceValue(value, path, seen)
case reflect.Array:
return copyPublicJSONSequenceValue(value, path, seen)
return copySequenceValue(value, path, seen)
default:
return nil, fmt.Errorf("%s: unsupported JSON value type %s", path, value.Type())
}
}
func copyPublicJSONMapValue(value reflect.Value, path string, seen map[jsonVisit]struct{}) (any, error) {
func copyMapValue(value reflect.Value, path string, seen map[visit]struct{}) (any, error) {
if value.IsNil() {
return nil, nil
}
@@ -103,37 +111,43 @@ func copyPublicJSONMapValue(value reflect.Value, path string, seen map[jsonVisit
return nil, fmt.Errorf("%s: map key type %s is not supported", path, value.Type().Key())
}
visit := jsonVisit{typ: value.Type(), ptr: value.Pointer()}
if _, ok := seen[visit]; ok {
current := visit{typ: value.Type(), ptr: value.Pointer()}
if _, ok := seen[current]; ok {
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
}
seen[visit] = struct{}{}
defer delete(seen, visit)
seen[current] = struct{}{}
defer delete(seen, current)
keys := value.MapKeys()
sort.Slice(keys, func(i, j int) bool {
return keys[i].String() < keys[j].String()
})
type entry struct {
key reflect.Value
name string
value any
}
entries := make([]entry, 0, value.Len())
entries := make([]entry, 0, len(keys))
preserveType := true
elemType := value.Type().Elem()
iter := value.MapRange()
for iter.Next() {
key := iter.Key()
elementType := value.Type().Elem()
for _, key := range keys {
name := key.String()
copied, err := copyPublicJSONValue(iter.Value(), path+"."+name, seen)
if name == "" {
return nil, fmt.Errorf("%s: map key must not be empty", path)
}
copied, err := copyValue(value.MapIndex(key), path+"."+name, seen)
if err != nil {
return nil, err
}
entries = append(entries, entry{key: key, name: name, value: copied})
if copied == nil {
if !canAssignNil(elemType) {
if !canAssignNil(elementType) {
preserveType = false
}
continue
}
if !reflect.TypeOf(copied).AssignableTo(elemType) {
if !reflect.TypeOf(copied).AssignableTo(elementType) {
preserveType = false
}
}
@@ -142,7 +156,7 @@ func copyPublicJSONMapValue(value reflect.Value, path string, seen map[jsonVisit
out := reflect.MakeMapWithSize(value.Type(), len(entries))
for _, entry := range entries {
if entry.value == nil {
out.SetMapIndex(entry.key, reflect.Zero(elemType))
out.SetMapIndex(entry.key, reflect.Zero(elementType))
continue
}
out.SetMapIndex(entry.key, reflect.ValueOf(entry.value))
@@ -157,33 +171,33 @@ func copyPublicJSONMapValue(value reflect.Value, path string, seen map[jsonVisit
return out, nil
}
func copyPublicJSONSequenceValue(value reflect.Value, path string, seen map[jsonVisit]struct{}) (any, error) {
var visit jsonVisit
func copySequenceValue(value reflect.Value, path string, seen map[visit]struct{}) (any, error) {
var current visit
if value.Kind() == reflect.Slice {
visit = jsonVisit{typ: value.Type(), ptr: value.Pointer()}
if _, ok := seen[visit]; ok {
current = visit{typ: value.Type(), ptr: value.Pointer()}
if _, ok := seen[current]; ok {
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
}
seen[visit] = struct{}{}
defer delete(seen, visit)
seen[current] = struct{}{}
defer delete(seen, current)
}
values := make([]any, value.Len())
preserveType := true
elemType := value.Type().Elem()
elementType := value.Type().Elem()
for i := 0; i < value.Len(); i++ {
copied, err := copyPublicJSONValue(value.Index(i), fmt.Sprintf("%s[%d]", path, i), seen)
copied, err := copyValue(value.Index(i), fmt.Sprintf("%s[%d]", path, i), seen)
if err != nil {
return nil, err
}
values[i] = copied
if copied == nil {
if !canAssignNil(elemType) {
if !canAssignNil(elementType) {
preserveType = false
}
continue
}
if !reflect.TypeOf(copied).AssignableTo(elemType) {
if !reflect.TypeOf(copied).AssignableTo(elementType) {
preserveType = false
}
}
@@ -195,7 +209,7 @@ func copyPublicJSONSequenceValue(value reflect.Value, path string, seen map[json
}
for i, copied := range values {
if copied == nil {
out.Index(i).Set(reflect.Zero(elemType))
out.Index(i).Set(reflect.Zero(elementType))
continue
}
out.Index(i).Set(reflect.ValueOf(copied))

View File

@@ -0,0 +1,97 @@
package jsonvalue_test
import (
"encoding/json"
"math"
"reflect"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
)
func TestCopyMapPreservesTypesAndIsolatesMutations(t *testing.T) {
nested := map[string]int{"limit": 2}
sequence := []string{"one", "two"}
input := map[string]any{
"count": int64(7),
"number": json.Number("-1.25e+2"),
"nested": nested,
"sequence": sequence,
}
copied, err := jsonvalue.CopyMap(input)
if err != nil {
t.Fatalf("copy map: %v", err)
}
nested["limit"] = 99
sequence[0] = "changed"
input["added"] = true
if got, ok := copied["count"].(int64); !ok || got != 7 {
t.Fatalf("integer type or value changed: %#v", copied["count"])
}
if got, ok := copied["number"].(json.Number); !ok || got != "-1.25e+2" {
t.Fatalf("JSON number type or value changed: %#v", copied["number"])
}
if got := copied["nested"].(map[string]int)["limit"]; got != 2 {
t.Fatalf("nested map was not isolated: %d", got)
}
if got := copied["sequence"].([]string)[0]; got != "one" {
t.Fatalf("sequence was not isolated: %q", got)
}
if _, ok := copied["added"]; ok {
t.Fatalf("top-level map was not isolated: %#v", copied)
}
}
func TestCopyMapRejectsInvalidValues(t *testing.T) {
cyclicMap := map[string]any{}
cyclicMap["self"] = cyclicMap
cyclicSlice := []any{nil}
cyclicSlice[0] = cyclicSlice
tests := []struct {
name string
value any
}{
{name: "empty nested key", value: map[string]int{"": 1}},
{name: "non-string map key", value: map[int]string{1: "one"}},
{name: "unsupported value", value: make(chan int)},
{name: "cyclic map", value: cyclicMap},
{name: "cyclic slice", value: cyclicSlice},
{name: "NaN", value: math.NaN()},
{name: "positive infinity", value: math.Inf(1)},
{name: "unsafe signed integer", value: int64(1 << 53)},
{name: "unsafe unsigned integer", value: uint64(1 << 53)},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
if _, err := jsonvalue.CopyMap(map[string]any{"value": tc.value}); err == nil {
t.Fatal("expected validation error")
}
})
}
}
func TestCopyMapValidatesJSONNumberSyntaxAndRange(t *testing.T) {
for _, number := range []json.Number{"0", "-1", "1.25", "-1.25e+2"} {
t.Run("valid "+number.String(), func(t *testing.T) {
got, err := jsonvalue.CopyMap(map[string]any{"value": number})
if err != nil {
t.Fatalf("copy valid JSON number: %v", err)
}
if !reflect.DeepEqual(got["value"], number) {
t.Fatalf("JSON number changed: got %#v want %#v", got["value"], number)
}
})
}
for _, number := range []json.Number{"", "01", "+1", "1.", ".1", "1e9999", "not-a-number"} {
t.Run("invalid "+number.String(), func(t *testing.T) {
if _, err := jsonvalue.CopyMap(map[string]any{"value": number}); err == nil {
t.Fatal("expected invalid JSON number error")
}
})
}
}

View File

@@ -12,7 +12,6 @@ import (
"os"
"strings"
"time"
"unicode/utf8"
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
@@ -177,12 +176,11 @@ func openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultMod
wireReq := openAIChatRequest{
Model: model,
}
if sessionID := strings.TrimSpace(req.Prompt.SessionID); sessionID != "" {
if n := utf8.RuneCountInString(sessionID); n > domain.SessionIDMaxLength {
return openAIChatRequest{}, fmt.Errorf("session_id length %d exceeds maximum %d", n, domain.SessionIDMaxLength)
}
wireReq.SessionID = sessionID
sessionID, err := domain.NormalizeSessionID(req.Prompt.SessionID)
if err != nil {
return openAIChatRequest{}, err
}
wireReq.SessionID = sessionID
wireReq.Messages = make([]openAIChatRequestMessage, 0, len(req.Prompt.Messages))
for _, msg := range req.Prompt.Messages {
@@ -262,7 +260,7 @@ func openAIChatRequestPayload(req openAIChatRequest) (map[string]any, error) {
if key == "" {
return nil, errors.New("extra_params key must not be empty")
}
if _, reserved := reservedOpenAIChatRequestFields[key]; reserved {
if IsReservedOpenAIChatRequestField(key) {
return nil, fmt.Errorf("extra_params key %q collides with reserved request field", key)
}
if _, err := json.Marshal(value); err != nil {
@@ -274,16 +272,23 @@ func openAIChatRequestPayload(req openAIChatRequest) (map[string]any, error) {
return out, nil
}
var reservedOpenAIChatRequestFields = map[string]struct{}{
"model": {},
"session_id": {},
"messages": {},
"temperature": {},
"max_tokens": {},
"top_p": {},
"service_tier": {},
"reasoning_effort": {},
"response_format": {},
// IsReservedOpenAIChatRequestField reports whether name is owned by the
// standard OpenAI-compatible chat request rather than extra parameters.
func IsReservedOpenAIChatRequestField(name string) bool {
switch name {
case "model",
"session_id",
"messages",
"temperature",
"max_tokens",
"top_p",
"service_tier",
"reasoning_effort",
"response_format":
return true
default:
return false
}
}
type openAIChatRequestMessage struct {

View File

@@ -1,9 +1,8 @@
id: aion-2
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: aion-labs/aion-2.0
temperature: 0.72
reasoning_effort: high
top_p: 0.95
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,7 +1,6 @@
id: claude-fable-latest
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "~anthropic/claude-fable-latest"
reasoning_effort: high
timeout_seconds: 600
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,7 +1,6 @@
id: claude-haiku-latest
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "~anthropic/claude-haiku-latest"
reasoning_effort: medium
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,7 +1,6 @@
id: claude-opus-latest
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "~anthropic/claude-opus-latest"
reasoning_effort: high
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,7 +1,6 @@
id: claude-sonnet-latest
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "~anthropic/claude-sonnet-latest"
reasoning_effort: high
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,7 +1,6 @@
id: deepseek-3-2
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: deepseek/deepseek-v3.2
reasoning_effort: high
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,7 +1,6 @@
id: deepseek-4-flash
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: deepseek/deepseek-v4-flash
#reasoning_effort: medium
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,7 +1,6 @@
id: deepseek-4-pro
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: deepseek/deepseek-v4-pro
reasoning_effort: high
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,9 +1,8 @@
id: gemini-2-flash-lite
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "google/gemini-2.5-flash-lite"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,9 +1,8 @@
id: gemini-2-flash
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "google/gemini-2.5-flash"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,9 +1,8 @@
id: gemini-2-pro
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "google/gemini-2.5-pro"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,9 +1,8 @@
id: gemini-3-flash-lite
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "google/gemini-3.1-flash-lite"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,9 +1,8 @@
id: gemini-flash-latest
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "~google/gemini-flash-latest"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,9 +1,8 @@
id: gemini-pro-latest
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "~google/gemini-pro-latest"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,9 +1,8 @@
id: gemma-4-31b
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: google/gemma-4-31b-it:exacto
temperature: 0.15
reasoning_effort: high
top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,9 +1,8 @@
id: minimax-m2
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: minimax/minimax-m2.5
temperature: 0.5
reasoning_effort: high
top_p: 0.95
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,9 +1,8 @@
id: minimax-m3
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: minimax/minimax-m3
#temperature: 0.5
reasoning_effort: high
#top_p: 0.95
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,7 +1,6 @@
id: mistral-large-2512
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: mistralai/mistral-large-2512
temperature: 0.15
top_p: 0.98
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY

View File

@@ -1,8 +1,7 @@
id: mistral-medium-3-5
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: mistralai/mistral-medium-3-5
temperature: 0.15
reasoning_effort: high
top_p: 0.98
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY

View File

@@ -1,7 +1,6 @@
id: mistral-small-3
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: mistralai/mistral-small-3.2-24b-instruct
temperature: 0.05
top_p: 1.0
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY

View File

@@ -1,8 +1,7 @@
id: mistral-small-4
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: mistralai/mistral-small-2603
temperature: 0.1
reasoning_effort: high
top_p: 0.98
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY

View File

@@ -1,7 +1,6 @@
id: nemotron-3-ultra
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: nvidia/nemotron-3-ultra-550b-a55b
reasoning_effort: high
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,7 +1,6 @@
id: gpt-5-mini
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "openai/gpt-5.4-mini"
reasoning_effort: high
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,7 +1,6 @@
id: gpt-5-nano
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "openai/gpt-5.4-nano"
reasoning_effort: high
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -7,6 +7,7 @@ import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
"gopkg.in/yaml.v3"
@@ -28,6 +29,12 @@ func TestBuiltInProfilesValidateThroughRepository(t *testing.T) {
if p.ID != id {
t.Fatalf("expected profile id %q, got %q", id, p.ID)
}
if p.BackendID != backend.OpenRouterID {
t.Fatalf("expected profile %q to select %q, got %q", id, backend.OpenRouterID, p.BackendID)
}
if p.Endpoint != "" || p.APIKeyEnv != "" {
t.Fatalf("expected profile %q to inherit backend connection settings, got endpoint=%q api_key_env=%q", id, p.Endpoint, p.APIKeyEnv)
}
})
}
}
@@ -60,6 +67,15 @@ func loadBuiltInProfileIDs(t *testing.T) map[string]string {
if _, ok := raw["api_key"]; ok {
t.Fatalf("built-in profile %s contains raw api_key", name)
}
if raw["backend"] != backend.OpenRouterID {
t.Fatalf("built-in profile %s does not select %q", name, backend.OpenRouterID)
}
if _, ok := raw["endpoint"]; ok {
t.Fatalf("built-in profile %s repeats endpoint", name)
}
if _, ok := raw["api_key_env"]; ok {
t.Fatalf("built-in profile %s repeats api_key_env", name)
}
id, ok := raw["id"].(string)
if !ok || strings.TrimSpace(id) == "" {
t.Fatalf("built-in profile %s has missing id", name)

View File

@@ -121,6 +121,7 @@ func loadProfile(ctx context.Context, fsys fs.FS, root string, id string) (*doma
if prof.ID != id {
continue
}
prof.BackendID = strings.TrimSpace(prof.BackendID)
if err := validateProfile(&prof); err != nil {
if errors.Is(err, ErrRawAPIKeyNotAllowed) {
return nil, fmt.Errorf("%w: %s", err, relPath)
@@ -189,8 +190,8 @@ func validateProfile(p *domain.ExecutionProfile) error {
if strings.TrimSpace(p.ID) == "" {
return errors.New("id is required")
}
if strings.TrimSpace(p.Endpoint) == "" {
return errors.New("endpoint is required")
if strings.TrimSpace(p.BackendID) == "" && strings.TrimSpace(p.Endpoint) == "" {
return errors.New("backend or endpoint is required")
}
if strings.TrimSpace(p.Model) == "" {
return errors.New("model is required")

View File

@@ -52,6 +52,43 @@ func TestFilesystemRepository_GetProfile(t *testing.T) {
}
})
t.Run("backend and endpoint connection matrix", func(t *testing.T) {
tests := []struct {
name string
connection string
wantBackend string
wantEndpoint string
wantErr bool
}{
{name: "backend only", connection: "backend: ' openrouter '", wantBackend: "openrouter"},
{name: "endpoint only", connection: "endpoint: http://localhost:8000/v1", wantEndpoint: "http://localhost:8000/v1"},
{name: "both", connection: "backend: openrouter\nendpoint: http://localhost:8000/v1", wantBackend: "openrouter", wantEndpoint: "http://localhost:8000/v1"},
{name: "neither", wantErr: true},
{name: "blank backend", connection: "backend: ' '", wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
id := "connection-" + strings.ReplaceAll(tt.name, " ", "-")
writeProfileTestFile(t, filepath.Join(tmpDir, id+".yaml"), "id: "+id+"\nmodel: model\n"+tt.connection+"\n")
p, err := repo.GetProfile(ctx, id)
if tt.wantErr {
if !errors.Is(err, ErrInvalidProfile) {
t.Fatalf("expected ErrInvalidProfile, got %v", err)
}
return
}
if err != nil {
t.Fatalf("expected profile to load, got %v", err)
}
if p.BackendID != tt.wantBackend || p.Endpoint != tt.wantEndpoint {
t.Fatalf("unexpected connection values: backend=%q endpoint=%q", p.BackendID, p.Endpoint)
}
})
}
})
t.Run("valid profile with api_key_env", func(t *testing.T) {
p, err := repo.GetProfile(ctx, "local-secure")
if err != nil {

View File

@@ -5,10 +5,9 @@ import (
"context"
"errors"
"fmt"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"strings"
"text/template"
"unicode/utf8"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
var (
@@ -95,10 +94,6 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
}
func renderSessionID(raw string, funcs template.FuncMap, vars map[string]string) (string, error) {
if strings.TrimSpace(raw) == "" {
return "", nil
}
tmpl, err := template.New("session_id").Funcs(funcs).Option("missingkey=error").Parse(raw)
if err != nil {
return "", fmt.Errorf("%w: session_id: %v", ErrInvalidTemplate, err)
@@ -109,9 +104,9 @@ func renderSessionID(raw string, funcs template.FuncMap, vars map[string]string)
return "", fmt.Errorf("%w: session_id: %w", ErrRenderFailure, err)
}
sessionID := strings.TrimSpace(buf.String())
if n := utf8.RuneCountInString(sessionID); n > domain.SessionIDMaxLength {
return "", fmt.Errorf("%w: session_id length %d exceeds maximum %d", ErrRenderFailure, n, domain.SessionIDMaxLength)
sessionID, err := domain.NormalizeSessionID(buf.String())
if err != nil {
return "", fmt.Errorf("%w: session_id: %v", ErrRenderFailure, err)
}
return sessionID, nil
}

View File

@@ -17,6 +17,7 @@ type OutputRepairer interface {
type RepairRequest struct {
PreviousOutput string
ValidationErrors []string
SessionID string
Target domain.ExecutionTarget
StructuredOutput *domain.StructuredOutputSpec
Attempt int
@@ -42,23 +43,26 @@ func (r *defaultOutputRepairer) Repair(ctx context.Context, req RepairRequest) (
errs = strings.Join(req.ValidationErrors, "\n")
}
prompt := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
{
Role: "system",
Content: "You repair invalid JSON output. Return only corrected JSON. Do not include explanations or markdown code fences.",
prompt := domain.RenderedPrompt{
SessionID: req.SessionID,
Messages: []domain.RenderedMessage{
{
Role: "system",
Content: "You repair invalid JSON output. Return only corrected JSON. Do not include explanations or markdown code fences.",
},
{
Role: "user",
Content: fmt.Sprintf(
"Repair attempt %d of %d for validation mode %s.\n\nValidation errors:\n%s\n\nPrevious output:\n%s\n\nReturn only corrected JSON.",
req.Attempt,
req.MaxAttempts,
req.Mode,
errs,
req.PreviousOutput,
),
},
},
{
Role: "user",
Content: fmt.Sprintf(
"Repair attempt %d of %d for validation mode %s.\n\nValidation errors:\n%s\n\nPrevious output:\n%s\n\nReturn only corrected JSON.",
req.Attempt,
req.MaxAttempts,
req.Mode,
errs,
req.PreviousOutput,
),
},
}}
}
resp, err := r.llm.Generate(ctx, domain.GenerateRequest{
Prompt: prompt,

View File

@@ -40,6 +40,7 @@ var (
type Runner struct {
promptDefs promptdef.Repository
profiles profile.Repository
backends BackendResolver
artifacts artifact.Reader
renderer prompt.Renderer
llm llm.Client
@@ -47,20 +48,27 @@ type Runner struct {
repairer OutputRepairer
}
// BackendResolver resolves one normalized backend ID.
type BackendResolver interface {
GetBackend(string) (domain.Backend, error)
}
func NewRunner(
promptDefs promptdef.Repository,
profiles profile.Repository,
backends BackendResolver,
artifacts artifact.Reader,
renderer prompt.Renderer,
llmClient llm.Client,
validator validate.Validator,
) *Runner {
return NewRunnerWithRepairer(promptDefs, profiles, artifacts, renderer, llmClient, validator, nil)
return NewRunnerWithRepairer(promptDefs, profiles, backends, artifacts, renderer, llmClient, validator, nil)
}
func NewRunnerWithRepairer(
promptDefs promptdef.Repository,
profiles profile.Repository,
backends BackendResolver,
artifacts artifact.Reader,
renderer prompt.Renderer,
llmClient llm.Client,
@@ -70,6 +78,7 @@ func NewRunnerWithRepairer(
return &Runner{
promptDefs: promptDefs,
profiles: profiles,
backends: backends,
artifacts: artifacts,
renderer: renderer,
llm: llmClient,
@@ -118,6 +127,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
repairResp, repairErr := r.repairer.Repair(ctx, RepairRequest{
PreviousOutput: genResp.Content,
ValidationErrors: validationResult.Errors,
SessionID: prepared.SessionID,
Target: prepared.EffectiveModelParams,
StructuredOutput: prepared.StructuredOutput,
Attempt: attemptsUsed,
@@ -151,8 +161,10 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
PromptID: prepared.PromptID,
PromptVersion: prepared.PromptVersion,
PromptHash: prepared.PromptHash,
SessionID: prepared.SessionID,
RenderedPromptHash: prepared.RenderedPromptHash,
SelectedProfileID: prepared.SelectedProfileID,
SelectedBackendID: prepared.SelectedBackendID,
ModelName: prepared.EffectiveModelParams.Model,
Endpoint: prepared.EffectiveModelParams.Endpoint,
EffectiveModelParams: prepared.EffectiveModelParams,
@@ -168,6 +180,10 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
if strings.TrimSpace(req.PromptID) == "" {
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
}
directSessionID, err := domain.NormalizeSessionID(req.SessionID)
if err != nil {
return nil, fmt.Errorf("%w: session_id: %v", ErrInvalidRequest, err)
}
start := time.Now().UTC()
@@ -193,7 +209,20 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
}
effectiveModel, targetPresence, err := resolveExecutionTarget(execProfile, req.Execution)
var selectedBackend *domain.Backend
if backendID := strings.TrimSpace(execProfile.BackendID); backendID != "" {
execProfile.BackendID = backendID
if r.backends == nil {
return nil, fmt.Errorf("%w: backend %q cannot be resolved", ErrProfileLoad, backendID)
}
resolvedBackend, resolveErr := r.backends.GetBackend(backendID)
if resolveErr != nil {
return nil, fmt.Errorf("%w: backend %q: %w", ErrProfileLoad, backendID, resolveErr)
}
selectedBackend = &resolvedBackend
}
effectiveModel, targetPresence, err := resolveExecutionTarget(selectedBackend, execProfile, req.Execution)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
}
@@ -228,10 +257,19 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
inputHashes[name] = art.Hash
}
renderedPrompt, err := r.renderer.Render(ctx, def, resolvedInputs, req.Vars)
definitionToRender := def
if directSessionID != "" {
definitionCopy := *def
definitionCopy.SessionID = ""
definitionToRender = &definitionCopy
}
renderedPrompt, err := r.renderer.Render(ctx, definitionToRender, resolvedInputs, req.Vars)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrPromptRender, err)
}
if directSessionID != "" {
renderedPrompt.SessionID = directSessionID
}
end := time.Now().UTC()
return &domain.PreparedRun{
@@ -239,6 +277,7 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
PromptVersion: def.Version,
PromptHash: promptDefinitionHash,
SelectedProfileID: selectedProfileID,
SelectedBackendID: effectiveModel.BackendID,
EffectiveModelParams: effectiveModel,
TargetPresence: targetPresence,
OutputContract: effectiveContract,
@@ -338,7 +377,10 @@ func (r *Runner) shouldAttemptRepair(contract domain.OutputContract, validationR
func mergeExecutionTarget(base domain.ExecutionTarget, override domain.ExecutionTarget) domain.ExecutionTarget {
out := base
if override.Endpoint != "" {
if strings.TrimSpace(override.BackendID) != "" {
out.BackendID = override.BackendID
}
if strings.TrimSpace(override.Endpoint) != "" {
out.Endpoint = override.Endpoint
}
if override.Model != "" {
@@ -367,6 +409,7 @@ func mergeExecutionTarget(base domain.ExecutionTarget, override domain.Execution
}
if override.APIKeyRequired {
out.APIKeyRequired = true
out.APIKeyEnv = ""
}
if len(override.ExtraParams) > 0 {
out.ExtraParams = copyExtraParams(override.ExtraParams)
@@ -414,8 +457,8 @@ func mergeExecutionTargetOverride(base domain.ExecutionTarget, override domain.E
if strings.TrimSpace(override.ServiceTier) != "" {
out.ServiceTier = override.ServiceTier
}
if strings.TrimSpace(override.ReasoningEffort) != "" {
out.ReasoningEffort = override.ReasoningEffort
if override.ReasoningEffort != nil {
out.ReasoningEffort = strings.TrimSpace(*override.ReasoningEffort)
}
if strings.TrimSpace(override.APIKeyEnv) != "" {
out.APIKeyEnv = override.APIKeyEnv
@@ -426,8 +469,9 @@ func mergeExecutionTargetOverride(base domain.ExecutionTarget, override domain.E
return out, presence, nil
}
func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) {
func resolveExecutionTarget(backendValue *domain.Backend, profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) {
out := defaults.ExecutionTargetDefault()
out = mergeExecutionTarget(out, backendToTarget(backendValue))
out = mergeExecutionTarget(out, executionProfileToTarget(profileValue))
var presence domain.ExecutionTargetPresence
if override != nil {
@@ -461,8 +505,13 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
if p == nil {
return domain.ExecutionTarget{}
}
endpoint := p.Endpoint
if strings.TrimSpace(endpoint) == "" {
endpoint = ""
}
return domain.ExecutionTarget{
Endpoint: p.Endpoint,
BackendID: p.BackendID,
Endpoint: endpoint,
Model: p.Model,
Temperature: p.Temperature,
MaxTokens: p.MaxTokens,
@@ -476,6 +525,18 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
}
}
func backendToTarget(value *domain.Backend) domain.ExecutionTarget {
if value == nil {
return domain.ExecutionTarget{}
}
return domain.ExecutionTarget{
BackendID: value.ID,
Endpoint: value.Endpoint,
APIKeyEnv: value.APIKeyEnv,
ExtraParams: copyExtraParams(value.ExtraParams),
}
}
func copyExtraParams(src map[string]any) map[string]any {
if len(src) == 0 {
return nil

View File

@@ -34,6 +34,18 @@ type fakeExecutionProfileRepo struct {
lastID string
}
type fakeBackendResolver struct {
backends map[string]domain.Backend
}
func (f fakeBackendResolver) GetBackend(id string) (domain.Backend, error) {
value, ok := f.backends[id]
if !ok {
return domain.Backend{}, errors.New("backend not found")
}
return value, nil
}
func (f *fakeExecutionProfileRepo) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
f.lastID = id
if f.err != nil {
@@ -167,7 +179,7 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) {
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
llmClient := &fakeLLM{forbid: true}
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
PromptVersion: "1",
@@ -210,6 +222,145 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) {
}
}
func TestRunnerDirectSessionResolution(t *testing.T) {
t.Run("direct value wins and changes only the rendered prompt hash", func(t *testing.T) {
def := promptDef(domain.FormatText, domain.ValidationNone, 0)
def.SessionID = "template-{{.template_session}}"
promptRepo := &fakePromptRepo{def: def}
runner := NewRunner(
promptRepo,
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
defaultArtifactReader(),
prompt.NewGoRenderer(),
&fakeLLM{forbid: true},
nil,
)
req := domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
Inputs: singleInputRef(),
Vars: map[string]string{"template_session": "from-template"},
}
req.SessionID = " direct-one "
first, err := runner.Prepare(context.Background(), req)
if err != nil {
t.Fatalf("prepare first direct session: %v", err)
}
req.SessionID = "direct-two"
second, err := runner.Prepare(context.Background(), req)
if err != nil {
t.Fatalf("prepare second direct session: %v", err)
}
if first.SessionID != "direct-one" || second.SessionID != "direct-two" {
t.Fatalf("direct sessions were not normalized: first=%q second=%q", first.SessionID, second.SessionID)
}
if first.PromptHash != second.PromptHash {
t.Fatalf("direct session changed prompt-definition hash: first=%q second=%q", first.PromptHash, second.PromptHash)
}
if first.RenderedPromptHash == second.RenderedPromptHash {
t.Fatal("changing direct session did not change rendered-prompt hash")
}
if def.SessionID != "template-{{.template_session}}" {
t.Fatalf("repository-owned prompt definition was mutated: %q", def.SessionID)
}
})
t.Run("direct value bypasses failing session template without changing messages", func(t *testing.T) {
def := promptDef(domain.FormatText, domain.ValidationNone, 0)
def.SessionID = "{{.missing_session}}"
def.Templates = []domain.PromptMessageTemplate{
{Role: "user", Content: "Hello {{.name}}"},
}
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
defaultArtifactReader(),
prompt.NewGoRenderer(),
&fakeLLM{forbid: true},
nil,
)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
SessionID: "direct-session",
Inputs: singleInputRef(),
Vars: map[string]string{"name": "Rin"},
})
if err != nil {
t.Fatalf("prepare with direct session: %v", err)
}
if prepared.SessionID != "direct-session" {
t.Fatalf("prepared session id = %q, want direct-session", prepared.SessionID)
}
if len(prepared.Messages) != 1 || prepared.Messages[0].Content != "Hello Rin" {
t.Fatalf("message templates did not render normally: %+v", prepared.Messages)
}
})
t.Run("blank direct value retains prompt template behavior", func(t *testing.T) {
def := promptDef(domain.FormatText, domain.ValidationNone, 0)
def.SessionID = " template-{{.template_session}} "
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
defaultArtifactReader(),
prompt.NewGoRenderer(),
&fakeLLM{forbid: true},
nil,
)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
SessionID: " \t ",
Inputs: singleInputRef(),
Vars: map[string]string{"template_session": "rendered"},
})
if err != nil {
t.Fatalf("prepare with prompt session template: %v", err)
}
if prepared.SessionID != "template-rendered" {
t.Fatalf("prepared session id = %q, want template-rendered", prepared.SessionID)
}
})
t.Run("overlong direct value fails before loading or generation", func(t *testing.T) {
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "unexpected"}}
runner := NewRunner(
promptRepo,
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
defaultArtifactReader(),
defaultRenderer(),
llmClient,
nil,
)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
SessionID: strings.Repeat("界", domain.SessionIDMaxLength+1),
Inputs: singleInputRef(),
})
if !errors.Is(err, ErrInvalidRequest) {
t.Fatalf("expected ErrInvalidRequest, got %v", err)
}
if promptRepo.lastID != "" {
t.Fatalf("invalid direct session loaded prompt %q", promptRepo.lastID)
}
if llmClient.calls != 0 {
t.Fatalf("invalid direct session invoked generation %d times", llmClient.calls)
}
})
}
func TestRunnerPrepareUsesPromptDefaultProfileWhenNoExplicitProfileID(t *testing.T) {
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
promptRepo.def.DefaultProfile = "from-prompt"
@@ -257,7 +408,7 @@ func TestRunnerPrepareSelectedProfileDoesNotExistFails(t *testing.T) {
}
func TestRunnerPreparePromptLoadFailure(t *testing.T) {
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p"})
if !errors.Is(err, ErrPromptLoad) {
t.Fatalf("expected ErrPromptLoad, got %v", err)
@@ -281,7 +432,7 @@ func TestRunnerPrepareRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) {
ServiceTier: "priority",
},
}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -379,12 +530,12 @@ func TestRunnerPrepareRequestNumericOverridePresence(t *testing.T) {
TopP: 0.8,
TimeoutSeconds: 45,
},
}},
}}, nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{forbid: true},
nil,
)
nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -426,12 +577,12 @@ func TestRunnerPrepareInvalidRequestNumericOverridesFail(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{forbid: true},
nil,
)
nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -458,7 +609,7 @@ func TestRunnerPrepareSelectedProfileBeatsBuiltInDefault(t *testing.T) {
ServiceTier: "priority",
},
}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -495,12 +646,12 @@ func TestRunnerPrepareFileBackedPromptBodiesRenderCorrectly(t *testing.T) {
llmClient := &fakeLLM{forbid: true}
runner := NewRunner(
promptdef.NewFilesystemRepository(promptDir),
profile.NewFilesystemRepository(profileDir),
profile.NewFilesystemRepository(profileDir), nil,
reader,
prompt.NewGoRenderer(),
llmClient,
nil,
)
nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "valid-file-backed",
@@ -526,12 +677,13 @@ func TestRunnerPrepareRequiredInputMissingFails(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
prompt.NewGoRenderer(),
&fakeLLM{forbid: true},
nil,
)
nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
@@ -552,12 +704,13 @@ func TestRunnerPrepareUnknownTemplateInputReferenceFails(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
prompt.NewGoRenderer(),
&fakeLLM{forbid: true},
nil,
)
nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
@@ -579,7 +732,7 @@ func TestRunnerPrepareAPIKeyEnvNameIncludedButNotResolvedValue(t *testing.T) {
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName},
}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if err != nil {
@@ -607,12 +760,12 @@ func TestRunnerPrepareJSONSchemaBuildsStructuredOutputSpec(t *testing.T) {
}
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{forbid: true},
validator,
)
validator)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -651,12 +804,12 @@ func TestRunnerPrepareJSONSchemaSchemaLoadFailureReturnsValidationError(t *testi
validator := &fakeValidator{schemaErr: errors.New("schema unavailable")}
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{forbid: true},
validator,
)
validator)
_, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -681,12 +834,12 @@ func TestRunnerRunJSONSchemaSchemaLoadFailureFailsBeforeLLM(t *testing.T) {
validator := &fakeValidator{schemaErr: errors.New("schema unavailable")}
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
defaultRenderer(),
llmClient,
validator,
)
validator)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -836,7 +989,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}}
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
PromptVersion: "1",
@@ -886,6 +1039,9 @@ func TestRunnerRunSuccessful(t *testing.T) {
if llmClient.lastReq.Prompt.SessionID != "session-123" {
t.Fatalf("expected session id to be sent to llm, got %q", llmClient.lastReq.Prompt.SessionID)
}
if res.SessionID != "session-123" {
t.Fatalf("expected session id in run result, got %q", res.SessionID)
}
if res.Usage.TotalTokens != 7 {
t.Fatalf("expected token usage to be retained, got %+v", res.Usage)
}
@@ -915,7 +1071,7 @@ func TestRunnerRunPassesExtraParamsToGenerateRequestTarget(t *testing.T) {
},
}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -941,7 +1097,7 @@ func TestRunnerRunAndPrepareResolveSameProfileAndEffectiveSettings(t *testing.T)
}}
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap"}}
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil)
req := domain.RunRequest{
PromptID: "p",
@@ -1071,7 +1227,7 @@ func TestRunnerRunExplicitRuntimeOverrideBeatsSelectedProfileValue(t *testing.T)
},
}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1115,7 +1271,7 @@ func TestRunnerRunSelectedProfileBeatsBuiltInDefault(t *testing.T) {
},
}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1142,7 +1298,7 @@ func TestRunnerRunBuiltInDefaultsUsedWhenProfileOmitsOptionalFields(t *testing.T
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"},
}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1172,7 +1328,7 @@ func TestRunnerRunAPIKeyEnvResolvesFromEnvironment(t *testing.T) {
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_TEST_API_KEY"},
}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if err != nil {
@@ -1188,7 +1344,7 @@ func TestRunnerRunAPIKeyEnvMissingEnvironmentValueFailsClearly(t *testing.T) {
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_MISSING_KEY"},
}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if !errors.Is(err, ErrInvalidRequest) {
@@ -1209,7 +1365,7 @@ func TestRunnerRunDirectAPIKeyBypassesMissingEnvAndReachesLLM(t *testing.T) {
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_MISSING_KEY"},
}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1233,7 +1389,7 @@ func TestRunnerPrepareAPIKeyRequiredFailsWithoutDirectKey(t *testing.T) {
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true},
}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1252,7 +1408,7 @@ func TestRunnerRunAPIKeyRequiredSucceedsWithDirectKey(t *testing.T) {
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true},
}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1279,7 +1435,7 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideWorks(t *testing.T) {
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"},
}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1304,7 +1460,7 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideBeatsProfile(t *testing.T) {
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: profileEnv},
}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1328,7 +1484,7 @@ func TestRunnerRunAPIKeyValueNotPresentInMetadata(t *testing.T) {
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName},
}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if err != nil {
@@ -1344,7 +1500,7 @@ func TestRunnerRunAPIKeyValueNotPresentInMetadata(t *testing.T) {
}
func TestRunnerRunPromptLoadFailure(t *testing.T) {
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"})
if !errors.Is(err, ErrPromptLoad) {
t.Fatalf("expected ErrPromptLoad, got %v", err)
@@ -1357,12 +1513,12 @@ func TestRunnerRunPromptLoadFailure(t *testing.T) {
func TestRunnerRunArtifactLoadFailure(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
&fakeArtifactReader{errByURI: map[string]error{"a://bad": errors.New("read failed")}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
nil,
)
nil)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1377,12 +1533,13 @@ func TestRunnerRunArtifactLoadFailure(t *testing.T) {
func TestRunnerRunPromptRenderFailure(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
&fakeRenderer{err: errors.New("render failed")},
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
nil,
)
nil)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
@@ -1396,12 +1553,13 @@ func TestRunnerRunPromptRenderFailure(t *testing.T) {
func TestRunnerRunLLMFailure(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{err: errors.New("llm failed")},
nil,
)
nil)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
@@ -1415,12 +1573,13 @@ func TestRunnerRunLLMFailure(t *testing.T) {
func TestRunnerRunCancellationPreservesGenerationCategory(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{resp: &domain.GenerateResponse{Content: "ignored"}},
nil,
)
nil)
ctx, cancel := context.WithCancel(context.Background())
cancel()
@@ -1440,12 +1599,13 @@ func TestRunnerRunCancellationPreservesGenerationCategory(t *testing.T) {
func TestRunnerRunLLMInvalidRequestMapsToUsecaseInvalidRequest(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{err: llm.ErrInvalidRequest},
nil,
)
nil)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
@@ -1463,12 +1623,13 @@ func TestRunnerRunValidationStillWorks(t *testing.T) {
validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}}
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}},
validator,
)
validator)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
@@ -1489,14 +1650,17 @@ func TestRunnerRunStructuredRepairRemainsBoundedAndUsesEffectiveModelSettings(t
runner := NewRunnerWithRepairer(
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", TimeoutSeconds: 55},
"exec": {ID: "exec", BackendID: "custom", Model: "profile-model", TimeoutSeconds: 55},
}}, fakeBackendResolver{backends: map[string]domain.Backend{
"custom": {ID: "custom", Endpoint: "http://backend/v1"},
}},
defaultArtifactReader(),
defaultRenderer(),
llmClient,
validate.NewStandardValidator("."),
repairer,
)
repairer)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
@@ -1518,6 +1682,47 @@ func TestRunnerRunStructuredRepairRemainsBoundedAndUsesEffectiveModelSettings(t
if repairer.reqs[0].Target.TimeoutSeconds != 22 {
t.Fatalf("expected repair to use effective timeout, got %d", repairer.reqs[0].Target.TimeoutSeconds)
}
if llmClient.lastReq.Target.BackendID != "custom" ||
repairer.reqs[0].Target.BackendID != "custom" ||
res.SelectedBackendID != "custom" {
t.Fatalf("expected backend identity in generation, repair, and result: generate=%q repair=%q result=%q",
llmClient.lastReq.Target.BackendID, repairer.reqs[0].Target.BackendID, res.SelectedBackendID)
}
}
func TestRunnerRunRepairCarriesEffectiveSessionID(t *testing.T) {
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: `{"broken":`}}
runner := NewRunnerWithRepairer(
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://example.test/v1", Model: "model"},
}},
nil,
defaultArtifactReader(),
defaultRenderer(),
llmClient,
validate.NewStandardValidator("."),
NewDefaultOutputRepairer(llmClient),
)
result, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
SessionID: " repair-session ",
Inputs: singleInputRef(),
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if llmClient.calls != 2 {
t.Fatalf("expected initial generation and one repair, got %d calls", llmClient.calls)
}
if llmClient.lastReq.Prompt.SessionID != "repair-session" {
t.Fatalf("expected repair generation to retain effective session, got %q", llmClient.lastReq.Prompt.SessionID)
}
if result.SessionID != "repair-session" {
t.Fatalf("expected result to retain effective session, got %q", result.SessionID)
}
}
func TestRunnerRunJSONSchemaRepairCarriesStructuredOutputSpec(t *testing.T) {
@@ -1546,13 +1751,13 @@ func TestRunnerRunJSONSchemaRepairCarriesStructuredOutputSpec(t *testing.T) {
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: `{"events":[1]}`}}
runner := NewRunnerWithRepairer(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
defaultRenderer(),
llmClient,
validator,
repairer,
)
repairer)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1629,13 +1834,12 @@ func TestResolveExecutionTargetProfileValuesPopulateAllSupportedFields(t *testin
ServiceTier: "priority",
ReasoningEffort: "low",
APIKeyEnv: "PROFILE_KEY",
APIKeyRequired: true,
ExtraParams: map[string]any{
"profile_option": "enabled",
},
}
target, presence, err := resolveExecutionTarget(profileValue, nil)
target, presence, err := resolveExecutionTarget(nil, profileValue, nil)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
@@ -1683,14 +1887,14 @@ func TestResolveExecutionTargetRuntimeOverridesBeatProfileForAllOverrideableFiel
TopP: float64Ptr(0.5),
TimeoutSeconds: intPtr(30),
ServiceTier: "flex",
ReasoningEffort: "high",
ReasoningEffort: stringPtr("high"),
APIKeyEnv: "RUNTIME_KEY",
ExtraParams: map[string]any{
"runtime_only": "yes",
},
}
target, presence, err := resolveExecutionTarget(profileValue, override)
target, presence, err := resolveExecutionTarget(nil, profileValue, override)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
@@ -1704,7 +1908,7 @@ func TestResolveExecutionTargetRuntimeOverridesBeatProfileForAllOverrideableFiel
target.TopP != *override.TopP ||
target.TimeoutSeconds != *override.TimeoutSeconds ||
target.ServiceTier != override.ServiceTier ||
target.ReasoningEffort != override.ReasoningEffort ||
target.ReasoningEffort != *override.ReasoningEffort ||
target.APIKeyEnv != override.APIKeyEnv {
t.Fatalf("expected runtime overrides to win for all fields, got %+v", target)
}
@@ -1713,6 +1917,46 @@ func TestResolveExecutionTargetRuntimeOverridesBeatProfileForAllOverrideableFiel
}
}
func TestResolveExecutionTargetReasoningOverrideStates(t *testing.T) {
profileValue := &domain.ExecutionProfile{
ReasoningEffort: "medium",
}
tests := []struct {
name string
override *string
want string
}{
{
name: "nil inherits profile value",
want: "medium",
},
{
name: "nonblank replaces and trims profile value",
override: stringPtr(" high "),
want: "high",
},
{
name: "blank clears profile value",
override: stringPtr(" \t "),
want: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
target, _, err := resolveExecutionTarget(nil, profileValue, &domain.ExecutionTargetOverride{
ReasoningEffort: tt.override,
})
if err != nil {
t.Fatalf("resolve execution target: %v", err)
}
if target.ReasoningEffort != tt.want {
t.Fatalf("reasoning effort = %q, want %q", target.ReasoningEffort, tt.want)
}
})
}
}
func TestMergeExecutionTargetEmptyStringOverridesDoNotErase(t *testing.T) {
base := domain.ExecutionTarget{
Endpoint: "http://base/v1",
@@ -1813,6 +2057,118 @@ func defaultExecutionProfile() *domain.ExecutionProfile {
}
}
func TestResolveExecutionTargetUsesBackendProfileAndRequestPrecedence(t *testing.T) {
backendValue := &domain.Backend{
ID: "custom",
Endpoint: "http://backend/v1",
APIKeyEnv: "BACKEND_KEY",
ExtraParams: map[string]any{"backend": true},
}
profileValue := &domain.ExecutionProfile{
ID: "exec",
BackendID: "custom",
Endpoint: "http://profile/v1",
Model: "profile-model",
APIKeyEnv: "PROFILE_KEY",
ExtraParams: map[string]any{"profile": true},
}
override := &domain.ExecutionTargetOverride{
Endpoint: "http://request/v1",
APIKeyEnv: "REQUEST_KEY",
ExtraParams: map[string]any{"request": true},
}
target, _, err := resolveExecutionTarget(backendValue, profileValue, override)
if err != nil {
t.Fatalf("resolve target: %v", err)
}
if target.BackendID != "custom" {
t.Fatalf("endpoint override changed backend identity: %+v", target)
}
if target.Endpoint != "http://request/v1" || target.APIKeyEnv != "REQUEST_KEY" {
t.Fatalf("request values did not win: %+v", target)
}
if !reflect.DeepEqual(target.ExtraParams, map[string]any{"request": true}) {
t.Fatalf("expected whole-map request replacement, got %#v", target.ExtraParams)
}
target, _, err = resolveExecutionTarget(backendValue, &domain.ExecutionProfile{
ID: "exec", BackendID: "custom", Model: "profile-model",
}, nil)
if err != nil {
t.Fatalf("resolve backend defaults: %v", err)
}
if target.Endpoint != backendValue.Endpoint ||
target.APIKeyEnv != backendValue.APIKeyEnv ||
!reflect.DeepEqual(target.ExtraParams, backendValue.ExtraParams) {
t.Fatalf("backend defaults were not inherited: %+v", target)
}
}
func TestRunnerPrepareBackendResolutionAndCredentialPrecedence(t *testing.T) {
resolver := fakeBackendResolver{backends: map[string]domain.Backend{
"custom": {ID: "custom", Endpoint: "http://backend/v1", APIKeyEnv: "BACKEND_KEY"},
}}
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
t.Run("unknown backend is a profile load failure", func(t *testing.T) {
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", BackendID: "unknown", Model: "model"},
}}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if !errors.Is(err, ErrProfileLoad) {
t.Fatalf("expected ErrProfileLoad, got %v", err)
}
})
t.Run("nil resolver is a profile load failure", func(t *testing.T) {
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", BackendID: "custom", Model: "model"},
}}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if !errors.Is(err, ErrProfileLoad) {
t.Fatalf("expected ErrProfileLoad, got %v", err)
}
})
t.Run("request environment wins", func(t *testing.T) {
t.Setenv("REQUEST_KEY", "request-secret")
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", BackendID: "custom", Model: "model", APIKeyEnv: "PROFILE_KEY"},
}}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(),
Execution: &domain.ExecutionTargetOverride{APIKeyEnv: "REQUEST_KEY"},
})
if err != nil {
t.Fatalf("prepare: %v", err)
}
if prepared.EffectiveModelParams.APIKeyEnv != "REQUEST_KEY" {
t.Fatalf("unexpected credential source: %+v", prepared.EffectiveModelParams)
}
})
t.Run("required direct key clears inherited environment", func(t *testing.T) {
t.Setenv("BACKEND_KEY", "backend-secret")
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", BackendID: "custom", Model: "model", APIKeyRequired: true},
}}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if !errors.Is(err, ErrAPIKeyRequired) {
t.Fatalf("expected ErrAPIKeyRequired, got %v", err)
}
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(), APIKey: "direct-secret",
})
if err != nil {
t.Fatalf("prepare with direct key: %v", err)
}
if prepared.EffectiveModelParams.APIKeyEnv != "" {
t.Fatalf("expected inherited environment name to be cleared, got %q", prepared.EffectiveModelParams.APIKeyEnv)
}
})
}
func defaultArtifactReader() *fakeArtifactReader {
return &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{
"a://ok": {Body: []byte("x"), Hash: hashString("x")},
@@ -1831,6 +2187,10 @@ func float64Ptr(v float64) *float64 {
return &v
}
func stringPtr(v string) *string {
return &v
}
func intPtr(v int) *int {
return &v
}
@@ -1838,10 +2198,11 @@ func intPtr(v int) *int {
func newMinimalRunner(promptRepo *fakePromptRepo, execRepo *fakeExecutionProfileRepo) *Runner {
return NewRunner(
promptRepo,
execRepo,
execRepo, nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
nil,
)
nil)
}

View File

@@ -26,6 +26,7 @@ func (r PreparedRun) MarshalJSON() ([]byte, error) {
PromptVersion string `json:"prompt_version,omitempty"`
PromptHash string `json:"prompt_hash,omitempty"`
SelectedProfileID string `json:"selected_profile_id"`
SelectedBackendID string `json:"selected_backend_id,omitempty"`
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
OutputContract OutputContract `json:"output_contract"`
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
@@ -41,6 +42,7 @@ func (r PreparedRun) MarshalJSON() ([]byte, error) {
PromptVersion: r.PromptVersion,
PromptHash: r.PromptHash,
SelectedProfileID: r.SelectedProfileID,
SelectedBackendID: r.SelectedBackendID,
EffectiveModelParams: r.EffectiveModelParams,
OutputContract: r.OutputContract,
StructuredOutput: r.StructuredOutput,
@@ -79,8 +81,10 @@ func (r RunResult) MarshalJSON() ([]byte, error) {
PromptID: r.PromptID,
PromptVersion: r.PromptVersion,
PromptHash: r.PromptHash,
SessionID: r.SessionID,
RenderedPromptHash: r.RenderedPromptHash,
SelectedProfileID: r.SelectedProfileID,
SelectedBackendID: r.SelectedBackendID,
ModelName: r.ModelName,
Endpoint: r.Endpoint,
EffectiveModelParams: r.EffectiveModelParams,
@@ -108,8 +112,10 @@ func (r *RunResult) UnmarshalJSON(data []byte) error {
PromptID: wire.PromptID,
PromptVersion: wire.PromptVersion,
PromptHash: wire.PromptHash,
SessionID: wire.SessionID,
RenderedPromptHash: wire.RenderedPromptHash,
SelectedProfileID: wire.SelectedProfileID,
SelectedBackendID: wire.SelectedBackendID,
ModelName: wire.ModelName,
Endpoint: wire.Endpoint,
EffectiveModelParams: wire.EffectiveModelParams,
@@ -134,8 +140,10 @@ type runResultJSON struct {
PromptID string `json:"prompt_id"`
PromptVersion string `json:"prompt_version,omitempty"`
PromptHash string `json:"prompt_hash,omitempty"`
SessionID string `json:"session_id,omitempty"`
RenderedPromptHash string `json:"rendered_prompt_hash"`
SelectedProfileID string `json:"selected_profile_id"`
SelectedBackendID string `json:"selected_backend_id,omitempty"`
ModelName string `json:"model_name"`
Endpoint string `json:"endpoint"`
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`

View File

@@ -7,6 +7,7 @@ import (
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
)
@@ -15,7 +16,8 @@ import (
//
// It does not register global state, maintain a model catalog, or resolve
// credentials. If APIKeyRequired is true, callers satisfy it with
// RunRequest.APIKey. Raw API keys do not belong in profiles.
// RunRequest.APIKey or an explicit request ExecutionTargetOverride.APIKeyEnv.
// Raw API keys do not belong in profiles.
//
// The function copies the ExtraParams map itself but does not recursively copy
// nested values. Validation and a deep copy occur when NewEngine applies a
@@ -23,6 +25,7 @@ import (
func OpenAICompatibleProfile(cfg OpenAICompatibleProfileConfig) Profile {
return Profile{
ID: cfg.ID,
BackendID: cfg.BackendID,
Endpoint: cfg.Endpoint,
Model: cfg.Model,
Temperature: cfg.Temperature,
@@ -79,12 +82,13 @@ func (r *memoryProfileRepository) GetProfile(_ context.Context, id string) (*dom
}
func toDomainProfile(publicProfile Profile) (domain.ExecutionProfile, error) {
extraParams, err := copyPublicJSONMap(publicProfile.ExtraParams)
extraParams, err := jsonvalue.CopyMap(publicProfile.ExtraParams)
if err != nil {
return domain.ExecutionProfile{}, err
}
prof := domain.ExecutionProfile{
ID: strings.TrimSpace(publicProfile.ID),
BackendID: strings.TrimSpace(publicProfile.BackendID),
Endpoint: publicProfile.Endpoint,
Model: publicProfile.Model,
Temperature: publicProfile.Temperature,
@@ -106,8 +110,8 @@ func validatePublicProfile(prof domain.ExecutionProfile) error {
if strings.TrimSpace(prof.ID) == "" {
return errors.New("id is required")
}
if strings.TrimSpace(prof.Endpoint) == "" {
return errors.New("endpoint is required")
if strings.TrimSpace(prof.BackendID) == "" && strings.TrimSpace(prof.Endpoint) == "" {
return errors.New("backend or endpoint is required")
}
if strings.TrimSpace(prof.Model) == "" {
return errors.New("model is required")

View File

@@ -3,6 +3,7 @@ package promptkit_test
import (
"context"
"encoding/json"
"errors"
"fmt"
"strings"
"sync"
@@ -26,6 +27,356 @@ func TestPreparedRunJSONOmitsZeroTimingValues(t *testing.T) {
}
}
func TestBackendIdentityJSONNamesAndOmission(t *testing.T) {
t.Run("execution target round trip", func(t *testing.T) {
value := promptkit.ExecutionTarget{BackendID: promptkit.BackendOpenRouter}
payload, err := json.Marshal(value)
if err != nil {
t.Fatalf("marshal execution target: %v", err)
}
var decoded promptkit.ExecutionTarget
if err := json.Unmarshal(payload, &decoded); err != nil {
t.Fatalf("unmarshal execution target: %v", err)
}
if decoded.BackendID != value.BackendID {
t.Fatalf("backend identity did not round trip: got %q want %q", decoded.BackendID, value.BackendID)
}
})
t.Run("prepared run round trip", func(t *testing.T) {
value := promptkit.PreparedRun{SelectedBackendID: promptkit.BackendOpenRouter}
payload, err := json.Marshal(value)
if err != nil {
t.Fatalf("marshal prepared run: %v", err)
}
var decoded promptkit.PreparedRun
if err := json.Unmarshal(payload, &decoded); err != nil {
t.Fatalf("unmarshal prepared run: %v", err)
}
if decoded.SelectedBackendID != value.SelectedBackendID {
t.Fatalf("backend identity did not round trip: got %q want %q", decoded.SelectedBackendID, value.SelectedBackendID)
}
})
t.Run("run result round trip", func(t *testing.T) {
value := promptkit.RunResult{SelectedBackendID: promptkit.BackendOpenRouter}
payload, err := json.Marshal(value)
if err != nil {
t.Fatalf("marshal run result: %v", err)
}
var decoded promptkit.RunResult
if err := json.Unmarshal(payload, &decoded); err != nil {
t.Fatalf("unmarshal run result: %v", err)
}
if decoded.SelectedBackendID != value.SelectedBackendID {
t.Fatalf("backend identity did not round trip: got %q want %q", decoded.SelectedBackendID, value.SelectedBackendID)
}
})
payload, err := json.Marshal(promptkit.ExecutionTarget{})
if err != nil {
t.Fatalf("marshal empty execution target: %v", err)
}
if strings.Contains(string(payload), `"backend_id"`) {
t.Fatalf("empty backend identity was not omitted: %s", payload)
}
}
func TestEndpointOnlyProfileOmitsBackendIdentityFromStableJSON(t *testing.T) {
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
promptkit.WithProfiles(promptkit.Profile{
ID: "profile", Endpoint: "http://example.test/v1", Model: "model",
}),
promptkit.WithLLMClient(client),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatalf("prepare endpoint-only profile: %v", err)
}
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatalf("run endpoint-only profile: %v", err)
}
if prepared.SelectedBackendID != "" ||
prepared.EffectiveModelParams.BackendID != "" ||
result.SelectedBackendID != "" ||
result.EffectiveModelParams.BackendID != "" {
t.Fatalf("endpoint-only profile acquired backend identity: prepared=%+v result=%+v", prepared, result)
}
for _, value := range []any{prepared, result} {
payload, err := json.Marshal(value)
if err != nil {
t.Fatalf("marshal endpoint-only value: %v", err)
}
if strings.Contains(string(payload), `"backend_id"`) || strings.Contains(string(payload), `"selected_backend_id"`) {
t.Fatalf("endpoint-only backend identity was not omitted: %s", payload)
}
}
}
func TestUnknownProfileBackendHasProfileLoadIdentity(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
promptkit.WithProfiles(promptkit.Profile{ID: "profile", BackendID: "unknown", Model: "model"}),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
_, err = engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if !errors.Is(err, promptkit.ErrProfileLoad) {
t.Fatalf("expected ErrProfileLoad, got %v", err)
}
if errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("unknown backend should not have invalid-request identity: %v", err)
}
}
func TestCustomBackendFlowsThroughProfilesOverridesAndInjectedClient(t *testing.T) {
t.Setenv("CUSTOM_LLM_KEY", "test-key")
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "backend-profile", "message"), "."),
promptkit.WithBackend(promptkit.Backend{
ID: " custom ",
Endpoint: " http://backend.example/v1 ",
APIKeyEnv: " CUSTOM_LLM_KEY ",
ExtraParams: map[string]any{
"provider": "custom",
},
}),
promptkit.WithProfiles(
promptkit.Profile{ID: "backend-profile", BackendID: "custom", Model: "backend-model"},
promptkit.Profile{ID: "profile-endpoint", BackendID: "custom", Endpoint: "http://profile.example/v1", Model: "profile-model"},
promptkit.Profile{ID: "blank-profile-endpoint", BackendID: "custom", Endpoint: " \t ", Model: "profile-model"},
),
promptkit.WithLLMClient(client),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatalf("run with custom backend: %v", err)
}
if len(client.requests) != 1 {
t.Fatalf("expected one injected-client request, got %d", len(client.requests))
}
target := client.requests[0].Target
if target.BackendID != "custom" ||
target.Endpoint != "http://backend.example/v1" ||
target.APIKeyEnv != "CUSTOM_LLM_KEY" ||
target.Model != "backend-model" ||
target.ExtraParams["provider"] != "custom" ||
result.SelectedBackendID != "custom" {
t.Fatalf("unexpected custom backend settings: target=%+v result_backend=%q", target, result.SelectedBackendID)
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: "prompt", ProfileID: "profile-endpoint",
})
if err != nil {
t.Fatalf("prepare profile endpoint override: %v", err)
}
if prepared.SelectedBackendID != "custom" || prepared.EffectiveModelParams.Endpoint != "http://profile.example/v1" {
t.Fatalf("profile endpoint override changed backend identity: %+v", prepared)
}
prepared, err = engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: "prompt", ProfileID: "blank-profile-endpoint",
})
if err != nil {
t.Fatalf("prepare blank profile endpoint: %v", err)
}
if prepared.SelectedBackendID != "custom" || prepared.EffectiveModelParams.Endpoint != "http://backend.example/v1" {
t.Fatalf("blank profile endpoint did not inherit backend endpoint: %+v", prepared)
}
prepared, err = engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: "prompt",
Execution: &promptkit.ExecutionTargetOverride{
Endpoint: "http://request.example/v1",
},
})
if err != nil {
t.Fatalf("prepare request endpoint override: %v", err)
}
if prepared.SelectedBackendID != "custom" || prepared.EffectiveModelParams.Endpoint != "http://request.example/v1" {
t.Fatalf("request endpoint override changed backend identity: %+v", prepared)
}
}
func TestCustomBackendSupportsFileProfileAndBothSelectionPaths(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "file-profile", "message"), "."),
promptkit.WithProfileFS(fstest.MapFS{
"profile.yaml": &fstest.MapFile{Data: []byte(`id: file-profile
backend: file-backend
endpoint: " "
model: file-model
`)},
}, "."),
promptkit.WithBackend(promptkit.Backend{
ID: "file-backend",
Endpoint: "http://file-backend.example/v1",
}),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
for _, request := range []promptkit.RunRequest{
{PromptID: "prompt"},
{PromptID: "prompt", ProfileID: "file-profile"},
} {
prepared, err := engine.Prepare(context.Background(), request)
if err != nil {
t.Fatalf("prepare file profile: %v", err)
}
if prepared.SelectedBackendID != "file-backend" ||
prepared.EffectiveModelParams.Endpoint != "http://file-backend.example/v1" {
t.Fatalf("unexpected file-profile backend resolution: %+v", prepared)
}
}
}
func TestBackendOptionsAccumulateAndRegistrationsAreEngineLocal(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "first-profile", "message"), "."),
promptkit.WithBackend(promptkit.Backend{ID: "first", Endpoint: "http://first.example/v1"}),
promptkit.WithBackend(promptkit.Backend{ID: "second", Endpoint: "http://second.example/v1"}),
promptkit.WithProfiles(
promptkit.Profile{ID: "first-profile", BackendID: "first", Model: "model"},
promptkit.Profile{ID: "second-profile", BackendID: "second", Model: "model"},
),
)
if err != nil {
t.Fatalf("construct engine with accumulated registrations: %v", err)
}
for profileID, wantEndpoint := range map[string]string{
"first-profile": "http://first.example/v1",
"second-profile": "http://second.example/v1",
} {
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: "prompt", ProfileID: profileID,
})
if err != nil {
t.Fatalf("prepare %s: %v", profileID, err)
}
if prepared.EffectiveModelParams.Endpoint != wantEndpoint {
t.Fatalf("profile %s endpoint=%q, want %q", profileID, prepared.EffectiveModelParams.Endpoint, wantEndpoint)
}
}
newEngine := func(endpoint string) *promptkit.Engine {
t.Helper()
value, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
promptkit.WithBackend(promptkit.Backend{ID: "same-id", Endpoint: endpoint}),
promptkit.WithProfiles(promptkit.Profile{ID: "profile", BackendID: "same-id", Model: "model"}),
)
if err != nil {
t.Fatalf("construct isolated engine: %v", err)
}
return value
}
firstEngine := newEngine("http://one.example/v1")
secondEngine := newEngine("http://two.example/v1")
for engine, wantEndpoint := range map[*promptkit.Engine]string{
firstEngine: "http://one.example/v1",
secondEngine: "http://two.example/v1",
} {
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatalf("prepare isolated engine: %v", err)
}
if prepared.EffectiveModelParams.Endpoint != wantEndpoint {
t.Fatalf("isolated engine endpoint=%q, want %q", prepared.EffectiveModelParams.Endpoint, wantEndpoint)
}
}
}
func TestBackendRegistrationRejectsInvalidAndDuplicateDefinitions(t *testing.T) {
cycle := map[string]any{}
cycle["self"] = cycle
tests := []struct {
name string
backends []promptkit.Backend
}{
{name: "blank id", backends: []promptkit.Backend{{Endpoint: "http://example.test/v1"}}},
{name: "invalid endpoint", backends: []promptkit.Backend{{ID: "custom", Endpoint: "ftp://example.test/v1"}}},
{name: "invalid environment", backends: []promptkit.Backend{{ID: "custom", Endpoint: "http://example.test/v1", APIKeyEnv: "BAD-NAME"}}},
{name: "reserved extra parameter", backends: []promptkit.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: map[string]any{"model": "override"}}}},
{name: "cyclic extra parameter", backends: []promptkit.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: cycle}}},
{name: "malformed JSON number", backends: []promptkit.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: map[string]any{"value": json.Number("01")}}}},
{name: "duplicate consumer id", backends: []promptkit.Backend{
{ID: " custom ", Endpoint: "http://one.example/v1"},
{ID: "custom", Endpoint: "http://two.example/v1"},
}},
{name: "reserved built-in id", backends: []promptkit.Backend{{
ID: promptkit.BackendOpenRouter, Endpoint: "http://replacement.example/v1",
}}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
options := []promptkit.Option{
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
}
for _, backend := range tt.backends {
options = append(options, promptkit.WithBackend(backend))
}
_, err := promptkit.NewEngine(promptkit.Config{}, options...)
if !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("expected ErrInvalidConfig, got %v", err)
}
})
}
}
func TestBackendExtraParamsAreDeeplyCopiedAtConstructionAndLookup(t *testing.T) {
nested := map[string]any{"value": "original"}
extraParams := map[string]any{"nested": nested}
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
promptkit.WithBackend(promptkit.Backend{
ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: extraParams,
}),
promptkit.WithProfiles(promptkit.Profile{ID: "profile", BackendID: "custom", Model: "model"}),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
nested["value"] = "mutated input"
extraParams["later"] = true
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatalf("first prepare: %v", err)
}
gotNested := prepared.EffectiveModelParams.ExtraParams["nested"].(map[string]any)
if gotNested["value"] != "original" || prepared.EffectiveModelParams.ExtraParams["later"] != nil {
t.Fatalf("backend retained caller mutations: %#v", prepared.EffectiveModelParams.ExtraParams)
}
gotNested["value"] = "mutated lookup"
prepared, err = engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatalf("second prepare: %v", err)
}
gotNested = prepared.EffectiveModelParams.ExtraParams["nested"].(map[string]any)
if gotNested["value"] != "original" {
t.Fatalf("backend retained lookup mutation: %#v", prepared.EffectiveModelParams.ExtraParams)
}
}
func TestPreparedRunJSONTimingRoundTrips(t *testing.T) {
start := time.Date(2026, time.July, 29, 12, 0, 0, 0, time.UTC)
prepared := promptkit.PreparedRun{
@@ -55,6 +406,7 @@ func TestRunResultJSONUsesMillisecondsAndRoundTrips(t *testing.T) {
result := promptkit.RunResult{
RunID: "opaque-run-id",
Artifact: promptkit.Artifact{Name: "output", ContentType: "text/plain", Body: []byte("ok")},
SessionID: "session-123",
StartTime: start,
EndTime: start.Add(1500 * time.Millisecond),
Duration: 1500 * time.Millisecond,
@@ -74,6 +426,9 @@ func TestRunResultJSONUsesMillisecondsAndRoundTrips(t *testing.T) {
if _, exists := object["duration"]; exists {
t.Fatalf("unexpected nanosecond duration field in %s", payload)
}
if got := object["session_id"]; got != result.SessionID {
t.Fatalf("expected session_id=%q, got %#v in %s", result.SessionID, got, payload)
}
artifact, ok := object["artifact"].(map[string]any)
if !ok || artifact["content_type"] != "text/plain" {
t.Fatalf("expected stable artifact JSON fields, got %#v", object["artifact"])
@@ -83,7 +438,10 @@ func TestRunResultJSONUsesMillisecondsAndRoundTrips(t *testing.T) {
if err := json.Unmarshal(payload, &decoded); err != nil {
t.Fatalf("unmarshal run result: %v", err)
}
if decoded.Duration != result.Duration || !decoded.StartTime.Equal(result.StartTime) || !decoded.EndTime.Equal(result.EndTime) {
if decoded.SessionID != result.SessionID ||
decoded.Duration != result.Duration ||
!decoded.StartTime.Equal(result.StartTime) ||
!decoded.EndTime.Equal(result.EndTime) {
t.Fatalf("timing values did not round trip: got %#v, want %#v", decoded, result)
}
@@ -91,11 +449,18 @@ func TestRunResultJSONUsesMillisecondsAndRoundTrips(t *testing.T) {
if err != nil {
t.Fatalf("marshal zero run result: %v", err)
}
for _, field := range []string{"start_time", "end_time", "duration_ms"} {
for _, field := range []string{"session_id", "start_time", "end_time", "duration_ms"} {
if strings.Contains(string(payload), `"`+field+`"`) {
t.Fatalf("expected zero %s to be omitted, got %s", field, payload)
}
}
var decodedEmpty promptkit.RunResult
if err := json.Unmarshal(payload, &decodedEmpty); err != nil {
t.Fatalf("unmarshal run result without session_id: %v", err)
}
if decodedEmpty.SessionID != "" {
t.Fatalf("expected absent session_id to decode empty, got %q", decodedEmpty.SessionID)
}
}
func TestEngineValidationIsSinglePass(t *testing.T) {
@@ -254,10 +619,14 @@ func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) {
func TestEngineSupportsConcurrentPrepareAndRun(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
promptkit.WithProfiles(promptkit.Profile{
ID: "profile",
promptkit.WithBackend(promptkit.Backend{
ID: "concurrent",
Endpoint: "http://example.test/v1",
Model: "model",
}),
promptkit.WithProfiles(promptkit.Profile{
ID: "profile",
BackendID: "concurrent",
Model: "model",
}),
promptkit.WithLLMClient(countingLLMClient{}),
)

104
types.go
View File

@@ -91,6 +91,16 @@ type RunRequest struct {
// profile is used; if both are empty, the error matches ErrProfileRequired
// and ErrInvalidRequest.
ProfileID string
// SessionID optionally supplies a direct per-run session identifier. A
// nonblank value is trimmed and overrides the prompt definition's
// session_id template. A blank value supplies no direct override. The
// maximum is 256 Unicode code points after trimming. A direct value is
// opaque consumer metadata, not a credential, and may be exposed in
// prepared values, results, collaborator requests, provider requests, and
// provider observability. Callers should use stable, non-sensitive
// identifiers. An overlong direct value makes Prepare or Run return an
// error matching ErrInvalidRequest.
SessionID string
// APIKey is a request-scoped direct credential. It takes precedence over
// APIKeyEnv, is passed to the selected LLMClient, and is never included in
// prepared values, results, hashes, JSON, String, or GoString output.
@@ -101,8 +111,8 @@ type RunRequest struct {
// Vars supplies Go-template data for messages and the session ID. Nil and
// empty maps are equivalent.
Vars map[string]string
// Execution optionally overrides individual profile execution settings.
// Nil uses the selected profile over framework defaults.
// Execution optionally overrides individual execution settings. Nil uses
// the selected profile over its backend, when any, and framework defaults.
Execution *ExecutionTargetOverride
// Validation optionally replaces the prompt's complete output contract. It
// does not merge individual fields. Nil uses the prompt contract.
@@ -126,8 +136,12 @@ type PreparedRun struct {
// SelectedProfileID is the explicit request profile or prompt default that
// supplied execution settings.
SelectedProfileID string `json:"selected_profile_id"`
// SelectedBackendID equals EffectiveModelParams.BackendID. It is empty for
// an endpoint-only profile.
SelectedBackendID string `json:"selected_backend_id,omitempty"`
// EffectiveModelParams contains framework defaults overlaid by the selected
// profile and then request overrides. It excludes resolved API-key values.
// backend, profile, and then request overrides. It excludes resolved API-key
// values.
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
// OutputContract is the complete effective output contract.
OutputContract OutputContract `json:"output_contract"`
@@ -136,7 +150,7 @@ type PreparedRun struct {
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
// InputHashes maps every supplied input name to its opaque artifact hash.
InputHashes map[string]string `json:"input_hashes,omitempty"`
// SessionID is the trimmed rendered session identifier, if any.
// SessionID is the effective direct or rendered session identifier, if any.
SessionID string `json:"session_id,omitempty"`
// RenderedPromptHash is an opaque equality value for SessionID and Messages.
RenderedPromptHash string `json:"rendered_prompt_hash"`
@@ -175,11 +189,17 @@ type RunResult struct {
// PromptHash is the same opaque definition equality value exposed by
// PreparedRun.
PromptHash string `json:"prompt_hash,omitempty"`
// SessionID is the effective direct or rendered session identifier, if any.
// JSON omits an empty value.
SessionID string `json:"session_id,omitempty"`
// RenderedPromptHash is the same opaque rendered-prompt equality value
// computed during preparation.
RenderedPromptHash string `json:"rendered_prompt_hash"`
// SelectedProfileID identifies the profile used for execution.
SelectedProfileID string `json:"selected_profile_id"`
// SelectedBackendID equals EffectiveModelParams.BackendID. It is empty for
// an endpoint-only profile.
SelectedBackendID string `json:"selected_backend_id,omitempty"`
// ModelName is the effective model name and equals
// EffectiveModelParams.Model.
ModelName string `json:"model_name"`
@@ -259,6 +279,11 @@ type ArtifactReader interface {
// ExecutionTarget represents effective model runtime settings and has a stable
// JSON representation. It never exposes a resolved API-key value.
type ExecutionTarget struct {
// BackendID is the effective routing identity selected by the profile. It
// remains unchanged when a profile or request overrides Endpoint and is
// empty for endpoint-only profiles. It is supplied to injected LLMClient
// implementations as part of the effective target.
BackendID string `json:"backend_id,omitempty"`
// Endpoint is the model-provider base URL.
Endpoint string `json:"endpoint"`
// Model is the provider model identifier.
@@ -276,7 +301,8 @@ type ExecutionTarget struct {
TimeoutSeconds int `json:"timeout_seconds"`
// ServiceTier is an optional provider-specific request tier.
ServiceTier string `json:"service_tier"`
// ReasoningEffort is an optional provider-specific reasoning setting.
// ReasoningEffort is the effective opaque provider-specific reasoning
// setting. An empty value instructs model clients to omit reasoning.
ReasoningEffort string `json:"reasoning_effort"`
// APIKeyEnv is an environment-variable name, not its credential value.
APIKeyEnv string `json:"api_key_env"`
@@ -287,13 +313,15 @@ type ExecutionTarget struct {
// ExecutionTargetOverride represents per-request runtime setting overrides and
// has no stable JSON representation.
//
// Non-empty string fields replace profile values. Non-nil numeric pointers
// replace profile values and preserve explicit zero. A non-empty ExtraParams
// map replaces the complete profile map rather than merging keys. Empty string
// fields, nil pointers, and a nil or empty ExtraParams map inherit the selected
// profile over framework defaults.
// Non-empty string fields replace profile and backend values. Non-nil pointer
// fields replace profile values and preserve explicit zero or empty values. A
// non-empty ExtraParams map replaces the complete profile or backend map
// rather than merging keys. Empty string fields, nil pointers, and a nil or
// empty ExtraParams map inherit the selected profile over its backend, when
// any, and framework defaults.
type ExecutionTargetOverride struct {
// Endpoint replaces the profile endpoint when non-empty.
// Endpoint replaces the profile or backend endpoint when non-empty without
// changing the effective BackendID.
Endpoint string
// Model replaces the profile model when non-empty.
Model string
@@ -308,24 +336,29 @@ type ExecutionTargetOverride struct {
TimeoutSeconds *int
// ServiceTier replaces the profile value when non-blank.
ServiceTier string
// ReasoningEffort replaces the profile value when non-blank. An empty value
// cannot clear a profile setting.
ReasoningEffort string
// APIKeyEnv replaces the profile environment-variable name when non-blank.
// A direct RunRequest.APIKey still takes precedence over environment lookup.
// ReasoningEffort controls the per-run reasoning setting. Nil inherits the
// profile value. A pointer to a non-blank string trims and replaces the
// profile value. A pointer to an empty or whitespace-only string clears the
// inherited value and disables reasoning for this run. Non-blank values
// are opaque and are not validated against a fixed vocabulary.
ReasoningEffort *string
// APIKeyEnv replaces the profile or backend environment-variable name when
// non-blank. A direct RunRequest.APIKey still takes precedence over
// environment lookup.
APIKeyEnv string
// ExtraParams, when non-empty, replaces the profile map. Values must be
// JSON-compatible: nil, booleans, finite numbers, strings, arrays or slices,
// and maps with non-empty string keys. Cycles are invalid.
// ExtraParams, when non-empty, replaces the complete profile or backend map.
// Values must be JSON-compatible: nil, booleans, finite numbers, strings,
// arrays or slices, and maps with non-empty string keys. Cycles are invalid.
ExtraParams map[string]any
}
// Profile is an in-memory execution profile for library consumers.
//
// It is equivalent to a loaded profile file after validation. Raw API keys do
// not belong in profiles; use APIKeyRequired to require callers to provide
// RunRequest.APIKey for each request, or use profile YAML api_key_env with file
// and FS profile sources. Profile has no stable JSON representation.
// not belong in profiles; use APIKeyRequired to require callers to provide a
// RunRequest.APIKey or explicit request ExecutionTargetOverride.APIKeyEnv, or
// use profile YAML api_key_env with file and FS profile sources. Profile has no
// stable JSON representation.
//
// WithProfiles validates and copies Profile values during NewEngine. Numeric
// zero, blank strings, and an empty ExtraParams map inherit framework defaults;
@@ -333,7 +366,13 @@ type ExecutionTargetOverride struct {
type Profile struct {
// ID is the required non-blank profile identifier. WithProfiles trims it.
ID string
// Endpoint is the required non-blank model-provider base URL.
// BackendID optionally selects an engine backend. WithProfiles trims it.
// Backend membership is checked when a request selects the profile; an
// unknown ID makes preparation fail with ErrProfileLoad.
BackendID string
// Endpoint is the model-provider base URL. It is required only when
// BackendID is blank and otherwise overrides the backend endpoint when
// non-blank.
Endpoint string
// Model is the required non-blank provider model identifier.
Model string
@@ -351,12 +390,13 @@ type Profile struct {
// ReasoningEffort is optional; a blank value inherits the framework
// default.
ReasoningEffort string
// APIKeyRequired requires a non-blank RunRequest.APIKey. It does not store a
// credential or enable environment lookup.
// APIKeyRequired clears a backend's inherited API-key environment name and
// requires a non-blank RunRequest.APIKey unless the request explicitly
// supplies ExecutionTargetOverride.APIKeyEnv. It does not store a credential.
APIKeyRequired bool
// ExtraParams contains provider-specific JSON-compatible values. An empty
// map inherits framework defaults. WithProfiles validates and deeply copies
// it during NewEngine.
// map inherits backend request defaults, when any. WithProfiles validates
// and deeply copies it during NewEngine.
ExtraParams map[string]any
}
@@ -364,13 +404,15 @@ type Profile struct {
// profile.
//
// It contains ordinary profile fields for OpenAI-compatible chat-completions
// endpoints. APIKeyRequired is satisfied by RunRequest.APIKey. Raw API keys do
// not belong in this config. OpenAICompatibleProfileConfig has no stable JSON
// endpoints. APIKeyRequired follows Profile.APIKeyRequired. Raw API keys do not
// belong in this config. OpenAICompatibleProfileConfig has no stable JSON
// representation and is not validated until its resulting Profile is supplied
// through WithProfiles to NewEngine.
type OpenAICompatibleProfileConfig struct {
// ID becomes Profile.ID.
ID string
// BackendID becomes Profile.BackendID.
BackendID string
// Endpoint becomes Profile.Endpoint.
Endpoint string
// Model becomes Profile.Model.
@@ -470,8 +512,8 @@ type TokenUsage struct {
// RenderedPrompt is the fully rendered prompt passed to an LLM client and has
// a stable JSON representation.
type RenderedPrompt struct {
// SessionID is the optional trimmed session identifier rendered from the
// prompt definition.
// SessionID is the optional effective direct or rendered session
// identifier supplied to the model client.
SessionID string `json:"session_id,omitempty"`
// Messages contains rendered messages in definition order.
Messages []RenderedMessage `json:"messages"`