Compare commits
11 Commits
b462153483
...
ffe6d261a9
| Author | SHA1 | Date | |
|---|---|---|---|
| ffe6d261a9 | |||
| dc39562ff7 | |||
| 0a839aa16d | |||
| f6ee18f6b3 | |||
| eb8ab215e8 | |||
| f89cb94ed2 | |||
| 359b7313f4 | |||
| ae210b3c26 | |||
| 810f80e7c9 | |||
| 8d00354c59 | |||
| d0010689f3 |
55
backends.go
Normal file
55
backends.go
Normal file
@@ -0,0 +1,55 @@
|
|||||||
|
package promptkit
|
||||||
|
|
||||||
|
import (
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
// BackendOpenRouter is the reserved ID of Promptkit's built-in OpenRouter
|
||||||
|
// backend.
|
||||||
|
const BackendOpenRouter = backend.OpenRouterID
|
||||||
|
|
||||||
|
// Backend configures one engine-scoped OpenAI-compatible backend.
|
||||||
|
//
|
||||||
|
// Backend has no stable JSON representation. Use keyed literals so additions
|
||||||
|
// to this configuration value do not break source compatibility.
|
||||||
|
type Backend struct {
|
||||||
|
// ID is the stable, case-sensitive registry key. NewEngine trims it and
|
||||||
|
// requires a non-blank value. BackendOpenRouter is reserved.
|
||||||
|
ID string
|
||||||
|
// Endpoint is the OpenAI-compatible base endpoint. NewEngine trims it and
|
||||||
|
// requires an absolute HTTP or HTTPS URL with a host and without user
|
||||||
|
// information, a query string, or a fragment. Paths are allowed.
|
||||||
|
Endpoint string
|
||||||
|
// APIKeyEnv optionally names the environment variable containing the API
|
||||||
|
// key. NewEngine trims it and requires the portable form
|
||||||
|
// [A-Za-z_][A-Za-z0-9_]*. Store only the name, never a credential value.
|
||||||
|
APIKeyEnv string
|
||||||
|
// ExtraParams contains backend-wide request defaults. Values must be
|
||||||
|
// JSON-compatible, finite, acyclic, and keyed by non-empty strings. Keys
|
||||||
|
// must not be model, session_id, messages, temperature, max_tokens, top_p,
|
||||||
|
// service_tier, reasoning_effort, or response_format. An empty map supplies
|
||||||
|
// no defaults. NewEngine deeply copies the map.
|
||||||
|
ExtraParams map[string]any
|
||||||
|
}
|
||||||
|
|
||||||
|
// WithBackend adds one Backend registration to the constructed Engine.
|
||||||
|
//
|
||||||
|
// Registrations accumulate in option order. Every normalized ID must be unique
|
||||||
|
// across consumer registrations and built-ins; a duplicate or invalid
|
||||||
|
// definition makes NewEngine fail with ErrInvalidConfig. In particular,
|
||||||
|
// BackendOpenRouter cannot be replaced. The immutable registration is scoped
|
||||||
|
// to the resulting Engine and cannot be enumerated, replaced, removed, or
|
||||||
|
// mutated after construction. WithBackend does not install package-global
|
||||||
|
// state.
|
||||||
|
func WithBackend(backend Backend) Option {
|
||||||
|
return optionFunc(func(options *engineOptions) error {
|
||||||
|
options.backends = append(options.backends, domain.Backend{
|
||||||
|
ID: backend.ID,
|
||||||
|
Endpoint: backend.Endpoint,
|
||||||
|
APIKeyEnv: backend.APIKeyEnv,
|
||||||
|
ExtraParams: backend.ExtraParams,
|
||||||
|
})
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
}
|
||||||
18
convert.go
18
convert.go
@@ -4,6 +4,7 @@ import (
|
|||||||
"reflect"
|
"reflect"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||||
)
|
)
|
||||||
|
|
||||||
func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
||||||
@@ -15,6 +16,7 @@ func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
|||||||
PromptID: req.PromptID,
|
PromptID: req.PromptID,
|
||||||
PromptVersion: req.PromptVersion,
|
PromptVersion: req.PromptVersion,
|
||||||
ProfileID: req.ProfileID,
|
ProfileID: req.ProfileID,
|
||||||
|
SessionID: req.SessionID,
|
||||||
APIKey: req.APIKey,
|
APIKey: req.APIKey,
|
||||||
Inputs: toDomainArtifactRefMap(req.Inputs),
|
Inputs: toDomainArtifactRefMap(req.Inputs),
|
||||||
Vars: copyStringMap(req.Vars),
|
Vars: copyStringMap(req.Vars),
|
||||||
@@ -32,6 +34,7 @@ func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
|
|||||||
PromptVersion: prepared.PromptVersion,
|
PromptVersion: prepared.PromptVersion,
|
||||||
PromptHash: prepared.PromptHash,
|
PromptHash: prepared.PromptHash,
|
||||||
SelectedProfileID: prepared.SelectedProfileID,
|
SelectedProfileID: prepared.SelectedProfileID,
|
||||||
|
SelectedBackendID: prepared.SelectedBackendID,
|
||||||
EffectiveModelParams: fromDomainExecutionTarget(prepared.EffectiveModelParams),
|
EffectiveModelParams: fromDomainExecutionTarget(prepared.EffectiveModelParams),
|
||||||
OutputContract: fromDomainOutputContract(prepared.OutputContract),
|
OutputContract: fromDomainOutputContract(prepared.OutputContract),
|
||||||
StructuredOutput: fromDomainStructuredOutputSpec(prepared.StructuredOutput),
|
StructuredOutput: fromDomainStructuredOutputSpec(prepared.StructuredOutput),
|
||||||
@@ -57,8 +60,10 @@ func fromDomainRunResult(result *domain.RunResult) *RunResult {
|
|||||||
PromptID: result.PromptID,
|
PromptID: result.PromptID,
|
||||||
PromptVersion: result.PromptVersion,
|
PromptVersion: result.PromptVersion,
|
||||||
PromptHash: result.PromptHash,
|
PromptHash: result.PromptHash,
|
||||||
|
SessionID: result.SessionID,
|
||||||
RenderedPromptHash: result.RenderedPromptHash,
|
RenderedPromptHash: result.RenderedPromptHash,
|
||||||
SelectedProfileID: result.SelectedProfileID,
|
SelectedProfileID: result.SelectedProfileID,
|
||||||
|
SelectedBackendID: result.SelectedBackendID,
|
||||||
ModelName: result.ModelName,
|
ModelName: result.ModelName,
|
||||||
Endpoint: result.Endpoint,
|
Endpoint: result.Endpoint,
|
||||||
EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams),
|
EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams),
|
||||||
@@ -131,7 +136,7 @@ func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain
|
|||||||
if override == nil {
|
if override == nil {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
extraParams, err := copyPublicJSONMap(override.ExtraParams)
|
extraParams, err := jsonvalue.CopyMap(override.ExtraParams)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -143,7 +148,7 @@ func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain
|
|||||||
TopP: copyFloat64Ptr(override.TopP),
|
TopP: copyFloat64Ptr(override.TopP),
|
||||||
TimeoutSeconds: copyIntPtr(override.TimeoutSeconds),
|
TimeoutSeconds: copyIntPtr(override.TimeoutSeconds),
|
||||||
ServiceTier: override.ServiceTier,
|
ServiceTier: override.ServiceTier,
|
||||||
ReasoningEffort: override.ReasoningEffort,
|
ReasoningEffort: copyStringPtr(override.ReasoningEffort),
|
||||||
APIKeyEnv: override.APIKeyEnv,
|
APIKeyEnv: override.APIKeyEnv,
|
||||||
ExtraParams: extraParams,
|
ExtraParams: extraParams,
|
||||||
}, nil
|
}, nil
|
||||||
@@ -151,6 +156,7 @@ func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain
|
|||||||
|
|
||||||
func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
|
func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
|
||||||
return ExecutionTarget{
|
return ExecutionTarget{
|
||||||
|
BackendID: target.BackendID,
|
||||||
Endpoint: target.Endpoint,
|
Endpoint: target.Endpoint,
|
||||||
Model: target.Model,
|
Model: target.Model,
|
||||||
Temperature: target.Temperature,
|
Temperature: target.Temperature,
|
||||||
@@ -396,6 +402,14 @@ func copyFloat64Ptr(src *float64) *float64 {
|
|||||||
return &v
|
return &v
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func copyStringPtr(src *string) *string {
|
||||||
|
if src == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
v := *src
|
||||||
|
return &v
|
||||||
|
}
|
||||||
|
|
||||||
func copyIntPtr(src *int) *int {
|
func copyIntPtr(src *int) *int {
|
||||||
if src == nil {
|
if src == nil {
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
21
doc.go
21
doc.go
@@ -2,9 +2,10 @@
|
|||||||
// prompt-defined LLM workflows.
|
// prompt-defined LLM workflows.
|
||||||
//
|
//
|
||||||
// Applications construct an [Engine] with [NewEngine], select filesystem or
|
// Applications construct an [Engine] with [NewEngine], select filesystem or
|
||||||
// in-memory sources with options, and call [Engine.Prepare] or [Engine.Run].
|
// in-memory sources and optional engine-scoped [Backend] registrations, and
|
||||||
// Concrete repositories, validators, and the built-in OpenAI-compatible client
|
// call [Engine.Prepare] or [Engine.Run]. Concrete registries, repositories,
|
||||||
// remain internal implementation details.
|
// validators, and the built-in OpenAI-compatible client remain internal
|
||||||
|
// implementation details.
|
||||||
//
|
//
|
||||||
// # Concurrency and ownership
|
// # Concurrency and ownership
|
||||||
//
|
//
|
||||||
@@ -12,11 +13,11 @@
|
|||||||
// or [ArtifactReader] can therefore receive concurrent calls and must be safe
|
// or [ArtifactReader] can therefore receive concurrent calls and must be safe
|
||||||
// for that use.
|
// for that use.
|
||||||
//
|
//
|
||||||
// NewEngine copies in-memory profiles. Prepare and Run copy request maps,
|
// NewEngine copies in-memory profiles and backend definitions. Prepare and Run
|
||||||
// slices, pointer values, and JSON-compatible extra parameters before using
|
// copy request maps, slices, pointer values, and JSON-compatible extra
|
||||||
// them. Returned values and values passed to extension interfaces are likewise
|
// parameters before using them. Returned values and values passed to extension
|
||||||
// isolated from engine state. Callers own those copies and may mutate them
|
// interfaces are likewise isolated from engine state. Callers own those copies
|
||||||
// after the call that supplied or returned them.
|
// and may mutate them after the call that supplied or returned them.
|
||||||
//
|
//
|
||||||
// # Security and sensitive data
|
// # Security and sensitive data
|
||||||
//
|
//
|
||||||
@@ -42,8 +43,8 @@
|
|||||||
// [GenerateResponse], [ExecutionTargetPresence], and the string value types
|
// [GenerateResponse], [ExecutionTargetPresence], and the string value types
|
||||||
// used by those values.
|
// used by those values.
|
||||||
//
|
//
|
||||||
// Construction values, including [Config], [RunRequest], [ArtifactRef],
|
// Construction values, including [Config], [Backend], [RunRequest],
|
||||||
// [ExecutionTargetOverride], [Profile], and
|
// [ArtifactRef], [ExecutionTargetOverride], [Profile], and
|
||||||
// [OpenAICompatibleProfileConfig], do not have stable JSON representations.
|
// [OpenAICompatibleProfileConfig], do not have stable JSON representations.
|
||||||
// Direct API keys are nevertheless excluded from JSON for every public value.
|
// Direct API keys are nevertheless excluded from JSON for every public value.
|
||||||
//
|
//
|
||||||
|
|||||||
@@ -33,8 +33,8 @@ engine, err := promptkit.NewEngine(promptkit.Config{
|
|||||||
})
|
})
|
||||||
```
|
```
|
||||||
|
|
||||||
Options support single-file or `fs.FS` sources, in-memory profiles, and
|
Options support single-file or `fs.FS` sources, in-memory profiles,
|
||||||
injected artifact or model clients. Consult the
|
engine-scoped backends, and injected artifact or model clients. Consult the
|
||||||
[constructor and option GoDoc](../../engine.go) for composition, precedence,
|
[constructor and option GoDoc](../../engine.go) for composition, precedence,
|
||||||
validation, and default transport behavior. Source discovery, format
|
validation, and default transport behavior. Source discovery, format
|
||||||
validation, and profile precedence are defined by the
|
validation, and profile precedence are defined by the
|
||||||
@@ -97,6 +97,66 @@ For programmatic profiles,
|
|||||||
[`OpenAICompatibleProfile`](../../profiles.go) converts ordinary
|
[`OpenAICompatibleProfile`](../../profiles.go) converts ordinary
|
||||||
OpenAI-compatible settings into a value accepted by `WithProfiles`.
|
OpenAI-compatible settings into a value accepted by `WithProfiles`.
|
||||||
|
|
||||||
|
### Set A Per-Run Session And Reasoning
|
||||||
|
|
||||||
|
Supply a direct session ID when one prompt should be correlated with a
|
||||||
|
consumer-managed conversation or workflow without changing prompt variables:
|
||||||
|
|
||||||
|
```go
|
||||||
|
reasoning := "high"
|
||||||
|
result, err := engine.Run(ctx, promptkit.RunRequest{
|
||||||
|
PromptID: "meeting.summary",
|
||||||
|
SessionID: "conversation-42",
|
||||||
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
|
"note": promptkit.Inline("Synthetic meeting notes"),
|
||||||
|
},
|
||||||
|
Execution: &promptkit.ExecutionTargetOverride{
|
||||||
|
ReasoningEffort: &reasoning,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
```
|
||||||
|
|
||||||
|
A nil reasoning pointer inherits the selected profile, a pointer to a
|
||||||
|
nonblank string replaces it, and a pointer to a blank string disables
|
||||||
|
reasoning for that run. Session IDs are correlation metadata, not credentials;
|
||||||
|
use stable, non-secret values that are safe to expose to collaborators and
|
||||||
|
providers. The
|
||||||
|
[`RunRequest` and `ExecutionTargetOverride` GoDoc](../../types.go) owns the
|
||||||
|
exact normalization, precedence, error, copying, and exposure contract.
|
||||||
|
|
||||||
|
### Register A Custom Backend
|
||||||
|
|
||||||
|
Register a reusable OpenAI-compatible connection once, then select it from a
|
||||||
|
profile:
|
||||||
|
|
||||||
|
```go
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
|
PromptDir: "prompts",
|
||||||
|
},
|
||||||
|
promptkit.WithBackend(promptkit.Backend{
|
||||||
|
ID: "local",
|
||||||
|
Endpoint: "http://localhost:8000/v1",
|
||||||
|
APIKeyEnv: "LOCAL_LLM_API_KEY",
|
||||||
|
}),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "local-summary",
|
||||||
|
BackendID: "local",
|
||||||
|
Model: "example-model",
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
Registrations belong to one engine and custom IDs cannot replace built-ins.
|
||||||
|
The [`Backend` and `WithBackend` GoDoc](../../backends.go) defines validation,
|
||||||
|
copying, uniqueness, and request-default behavior.
|
||||||
|
|
||||||
|
Both file-backed and in-memory profiles select a registration through
|
||||||
|
`backend` or `Profile.BackendID`. Profile and request endpoint overrides retain
|
||||||
|
that routing identity. `PreparedRun.SelectedBackendID`,
|
||||||
|
`RunResult.SelectedBackendID`, and the effective `ExecutionTarget.BackendID`
|
||||||
|
expose it to consumers and injected model clients. Endpoint-only profiles
|
||||||
|
remain supported and expose an empty backend ID.
|
||||||
|
|
||||||
## Credentials
|
## Credentials
|
||||||
|
|
||||||
File-backed profiles name an environment variable; in-memory profiles can
|
File-backed profiles name an environment variable; in-memory profiles can
|
||||||
@@ -137,7 +197,8 @@ distinguish invalid construction, invalid requests, absent sources,
|
|||||||
source-loading failures, collaborator failures, and operational validation
|
source-loading failures, collaborator failures, and operational validation
|
||||||
failures. Specific request conditions may also match the broader
|
failures. Specific request conditions may also match the broader
|
||||||
`ErrInvalidRequest`, and injected collaborator identities are preserved where
|
`ErrInvalidRequest`, and injected collaborator identities are preserved where
|
||||||
documented.
|
documented. Invalid or duplicate backend registrations match
|
||||||
|
`ErrInvalidConfig`; selecting an unknown backend matches `ErrProfileLoad`.
|
||||||
|
|
||||||
## Application Boundary
|
## Application Boundary
|
||||||
|
|
||||||
|
|||||||
@@ -91,7 +91,8 @@ of a named input. Missing variables and input references are errors.
|
|||||||
|
|
||||||
The optional `session_id` uses the same template data and input helper. Its
|
The optional `session_id` uses the same template data and input helper. Its
|
||||||
rendered value is trimmed, omitted when empty, and limited to 256 Unicode code
|
rendered value is trimmed, omitted when empty, and limited to 256 Unicode code
|
||||||
points.
|
points. A nonblank direct request session ID bypasses this template completely;
|
||||||
|
a blank direct value leaves the template behavior unchanged.
|
||||||
|
|
||||||
### Cache Control
|
### Cache Control
|
||||||
|
|
||||||
@@ -136,7 +137,7 @@ A profile supplies model execution settings:
|
|||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
id: local-summary
|
id: local-summary
|
||||||
endpoint: http://localhost:8000/v1
|
backend: openrouter
|
||||||
model: example-model
|
model: example-model
|
||||||
temperature: 0.2
|
temperature: 0.2
|
||||||
max_tokens: 500
|
max_tokens: 500
|
||||||
@@ -144,7 +145,6 @@ top_p: 0.95
|
|||||||
timeout_seconds: 90
|
timeout_seconds: 90
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
reasoning_effort: medium
|
reasoning_effort: medium
|
||||||
api_key_env: EXAMPLE_API_KEY
|
|
||||||
extra_params:
|
extra_params:
|
||||||
provider_option: enabled
|
provider_option: enabled
|
||||||
```
|
```
|
||||||
@@ -152,7 +152,8 @@ extra_params:
|
|||||||
| Field | Required | Meaning |
|
| Field | Required | Meaning |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| `id` | yes | Non-empty profile identifier. IDs must be unique within one source. |
|
| `id` | yes | Non-empty profile identifier. IDs must be unique within one source. |
|
||||||
| `endpoint` | yes | Non-empty OpenAI-compatible base URL, including an API version path when required. |
|
| `backend` | unless `endpoint` is present | Backend registry ID. It is trimmed and registry membership is checked when the profile is prepared. |
|
||||||
|
| `endpoint` | unless `backend` is present | Non-empty OpenAI-compatible base URL, including an API version path when required. When both connection fields are present, this overrides the backend endpoint without changing backend identity. |
|
||||||
| `model` | yes | Non-empty provider model name. |
|
| `model` | yes | Non-empty provider model name. |
|
||||||
| `temperature` | no | Number from 0 through 2. |
|
| `temperature` | no | Number from 0 through 2. |
|
||||||
| `max_tokens` | no | Integer zero or greater. |
|
| `max_tokens` | no | Integer zero or greater. |
|
||||||
@@ -166,6 +167,13 @@ extra_params:
|
|||||||
Raw `api_key` is prohibited in profile YAML. Store only an environment
|
Raw `api_key` is prohibited in profile YAML. Store only an environment
|
||||||
variable name in `api_key_env`.
|
variable name in `api_key_env`.
|
||||||
|
|
||||||
|
Promptkit does not infer a backend from a model or endpoint. Endpoint-only
|
||||||
|
profiles remain supported and have no effective backend ID.
|
||||||
|
The engine always provides the built-in `openrouter` ID. Consumers can add
|
||||||
|
engine-scoped IDs with
|
||||||
|
[`WithBackend`](../backends.go); exact registration validation belongs to its
|
||||||
|
GoDoc.
|
||||||
|
|
||||||
`extra_params` accepts null, booleans, finite numbers, strings, arrays, and
|
`extra_params` accepts null, booleans, finite numbers, strings, arrays, and
|
||||||
objects with string keys. Keys must be non-empty. With the built-in client,
|
objects with string keys. Keys must be non-empty. With the built-in client,
|
||||||
they also cannot collide with the standard fields listed in the
|
they also cannot collide with the standard fields listed in the
|
||||||
@@ -176,8 +184,9 @@ they also cannot collide with the standard fields listed in the
|
|||||||
Execution settings resolve in this order:
|
Execution settings resolve in this order:
|
||||||
|
|
||||||
1. framework defaults;
|
1. framework defaults;
|
||||||
2. the selected profile; and
|
2. the selected backend, when the profile names one;
|
||||||
3. request `ExecutionTargetOverride` values.
|
3. the selected profile; and
|
||||||
|
4. request `ExecutionTargetOverride` values.
|
||||||
|
|
||||||
The framework defaults are:
|
The framework defaults are:
|
||||||
|
|
||||||
@@ -194,8 +203,13 @@ explicit zero is preserved. In particular, an explicit request
|
|||||||
`timeout_seconds` of zero disables the per-generation deadline while leaving
|
`timeout_seconds` of zero disables the per-generation deadline while leaving
|
||||||
the caller context and transport timeout intact.
|
the caller context and transport timeout intact.
|
||||||
|
|
||||||
Non-empty request strings replace profile strings. A non-empty request
|
Non-empty profile strings replace backend defaults, and non-empty request
|
||||||
`ExtraParams` map replaces the profile map rather than merging keys.
|
strings replace both. Request reasoning is the exception: a nil
|
||||||
|
`ReasoningEffort` pointer inherits the profile, a pointer to a nonblank string
|
||||||
|
trims and replaces it, and a pointer to a blank string clears it. Backend
|
||||||
|
identity is retained when either layer overrides the endpoint. A non-empty
|
||||||
|
`extra_params` map at each layer replaces the entire lower-precedence map
|
||||||
|
rather than merging keys.
|
||||||
The [outbound integration contract](integrations/openai-compatible-chat.md)
|
The [outbound integration contract](integrations/openai-compatible-chat.md)
|
||||||
defines how the effective settings are serialized.
|
defines how the effective settings are serialized.
|
||||||
|
|
||||||
@@ -217,8 +231,11 @@ invalid matching profile is an error and does not fall back. In-memory
|
|||||||
|
|
||||||
## Built-In Profile Catalog
|
## Built-In Profile Catalog
|
||||||
|
|
||||||
Built-ins use the OpenRouter-compatible endpoint and
|
Every built-in selects the `openrouter` backend. The engine's built-in backend
|
||||||
`OPENROUTER_API_KEY`. A custom or in-memory profile with the same ID takes
|
registry supplies `https://openrouter.ai/api/v1` and the environment-variable
|
||||||
|
name `OPENROUTER_API_KEY`, so individual profiles contain only model and
|
||||||
|
generation settings. Built-in profile files do not repeat those connection
|
||||||
|
values. A custom or in-memory profile with the same profile ID takes
|
||||||
precedence.
|
precedence.
|
||||||
|
|
||||||
| Provider | ID | Model |
|
| Provider | ID | Model |
|
||||||
@@ -270,6 +287,10 @@ prompt, profile, schema, or example files:
|
|||||||
- a request can provide a direct `APIKey` or override `APIKeyEnv`; and
|
- a request can provide a direct `APIKey` or override `APIKeyEnv`; and
|
||||||
- a direct request key takes precedence over environment lookup.
|
- a direct request key takes precedence over environment lookup.
|
||||||
|
|
||||||
|
After a direct request key, the credential-source precedence is request
|
||||||
|
`APIKeyEnv`, profile `api_key_env`, then the backend default. An in-memory
|
||||||
|
profile with `APIKeyRequired` clears an inherited backend environment name and
|
||||||
|
requires a direct key unless the request explicitly supplies `APIKeyEnv`.
|
||||||
Promptkit validates required credential availability during preparation.
|
Promptkit validates required credential availability during preparation.
|
||||||
Direct keys are excluded from JSON results and redacted by public string
|
Direct keys are excluded from JSON results and redacted by public string
|
||||||
formatters. Environment-variable names may appear in prepared metadata, but
|
formatters. Environment-variable names may appear in prepared metadata, but
|
||||||
|
|||||||
@@ -13,10 +13,15 @@ that produce these outbound settings.
|
|||||||
## Endpoint And Method
|
## Endpoint And Method
|
||||||
|
|
||||||
Generation sends an HTTP `POST` with `Content-Type: application/json`.
|
Generation sends an HTTP `POST` with `Content-Type: application/json`.
|
||||||
A non-empty endpoint from the execution target overrides the client's
|
Before the client is called, the engine resolves framework, backend, profile,
|
||||||
configured base URL. After trailing slashes are removed,
|
and request values into one execution target. A non-empty endpoint from that
|
||||||
`/chat/completions` is appended. Generation fails before sending when neither
|
target overrides the client's configured base URL. After trailing slashes are
|
||||||
source supplies an endpoint.
|
removed, `/chat/completions` is appended. Generation fails before sending when
|
||||||
|
neither source supplies an endpoint.
|
||||||
|
|
||||||
|
The target's backend ID is routing metadata for prepared values, results, and
|
||||||
|
injected clients. The built-in client does not derive the URL from that ID and
|
||||||
|
does not serialize it in the provider request.
|
||||||
|
|
||||||
## Authentication
|
## Authentication
|
||||||
|
|
||||||
@@ -26,6 +31,12 @@ the client reads that variable and requires a non-empty value. The selected
|
|||||||
key is sent as `Authorization: Bearer <key>`. No authorization header is sent
|
key is sent as `Authorization: Bearer <key>`. No authorization header is sent
|
||||||
when neither mechanism is configured.
|
when neither mechanism is configured.
|
||||||
|
|
||||||
|
The target contains the already resolved environment-variable name: an
|
||||||
|
explicit request override takes precedence over profile metadata, which takes
|
||||||
|
precedence over the backend default. Only the name reaches prepared metadata;
|
||||||
|
the environment value is read just before the provider call and is never added
|
||||||
|
to the JSON body.
|
||||||
|
|
||||||
## Request Body
|
## Request Body
|
||||||
|
|
||||||
The request body always contains `model` and `messages`. The execution
|
The request body always contains `model` and `messages`. The execution
|
||||||
@@ -36,20 +47,24 @@ Each ordinary message contains its `role` and string `content`. A
|
|||||||
cache-controlled message instead uses a text content block containing `type`,
|
cache-controlled message instead uses a text content block containing `type`,
|
||||||
`text`, and `cache_control`; an empty cache-control TTL is omitted.
|
`text`, and `cache_control`; an empty cache-control TTL is omitted.
|
||||||
|
|
||||||
A non-empty session ID is trimmed, checked against the internal domain limit,
|
The effective direct or prompt-rendered session ID is trimmed, limited to 256
|
||||||
and sent as top-level `session_id`. It is not sent as a session header.
|
Unicode code points, and sent when nonempty as top-level `session_id`. It is
|
||||||
|
never also sent as a session header.
|
||||||
|
|
||||||
The client conditionally includes:
|
The client conditionally includes:
|
||||||
|
|
||||||
- `temperature`, `max_tokens`, and `top_p` when non-zero or explicitly
|
- `temperature`, `max_tokens`, and `top_p` when non-zero or explicitly
|
||||||
present;
|
present;
|
||||||
- non-empty `service_tier` and `reasoning_effort`; and
|
- non-empty `service_tier` and effective `reasoning_effort`; an explicitly
|
||||||
|
disabled reasoning setting is empty and therefore omitted; and
|
||||||
- `response_format` for JSON Schema structured output, including its name,
|
- `response_format` for JSON Schema structured output, including its name,
|
||||||
strict flag, and schema document.
|
strict flag, and schema document.
|
||||||
|
|
||||||
Extra parameters are merged directly into the top-level body after JSON
|
The engine resolves backend, profile, and request extra-parameter maps by
|
||||||
serialization is verified. Empty keys and collisions with these reserved
|
whole-map replacement rather than key merging. The resulting effective map is
|
||||||
fields are rejected before any provider call:
|
then merged directly into the top-level body after JSON serialization is
|
||||||
|
verified. Empty keys and collisions with these reserved fields are rejected
|
||||||
|
before any provider call:
|
||||||
|
|
||||||
- `model`
|
- `model`
|
||||||
- `session_id`
|
- `session_id`
|
||||||
@@ -61,6 +76,9 @@ fields are rejected before any provider call:
|
|||||||
- `reasoning_effort`
|
- `reasoning_effort`
|
||||||
- `response_format`
|
- `response_format`
|
||||||
|
|
||||||
|
`backend_id`, `api_key_env`, and resolved credential values are not provider
|
||||||
|
request fields.
|
||||||
|
|
||||||
## Response Handling
|
## Response Handling
|
||||||
|
|
||||||
Any 2xx response is decoded as an OpenAI-compatible chat response. The client
|
Any 2xx response is decoded as an OpenAI-compatible chat response. The client
|
||||||
|
|||||||
@@ -20,6 +20,11 @@ orchestration. `OpenAICompatibleClient` is the built-in implementation. It
|
|||||||
uses internal domain values for rendered prompts, execution targets,
|
uses internal domain values for rendered prompts, execution targets,
|
||||||
structured output, responses, and token usage.
|
structured output, responses, and token usage.
|
||||||
|
|
||||||
|
The runner supplies a fully resolved target after applying backend, profile,
|
||||||
|
and request precedence. The client uses its endpoint, credential metadata,
|
||||||
|
generation fields, and extra parameters. `BackendID` remains routing metadata
|
||||||
|
for the generation boundary and is not mapped into the provider payload.
|
||||||
|
|
||||||
Construction validates the configured base URL and clones any supplied
|
Construction validates the configured base URL and clones any supplied
|
||||||
`http.Client` so Promptkit can apply its timeout default without mutating the
|
`http.Client` so Promptkit can apply its timeout default without mutating the
|
||||||
caller's client. Generation then:
|
caller's client. Generation then:
|
||||||
@@ -31,6 +36,10 @@ caller's client. Generation then:
|
|||||||
5. performs the outbound request under the applicable deadlines; and
|
5. performs the outbound request under the applicable deadlines; and
|
||||||
6. decodes the first response choice and token usage.
|
6. decodes the first response choice and token usage.
|
||||||
|
|
||||||
|
`internal/llm` owns the set of reserved OpenAI-compatible request fields used
|
||||||
|
when validating extra parameters. Backend registration consumes the same rule
|
||||||
|
without making the model client depend on registry configuration.
|
||||||
|
|
||||||
The implementation has no retry loop, tool-call support, provider catalog,
|
The implementation has no retry loop, tool-call support, provider catalog,
|
||||||
inbound HTTP behavior, or durable session store.
|
inbound HTTP behavior, or durable session store.
|
||||||
|
|
||||||
@@ -51,5 +60,7 @@ The
|
|||||||
[OpenAI-compatible client tests](../../internal/llm/openai_compatible_client_test.go)
|
[OpenAI-compatible client tests](../../internal/llm/openai_compatible_client_test.go)
|
||||||
own configuration, client cloning, deterministic deadline precedence,
|
own configuration, client cloning, deterministic deadline precedence,
|
||||||
authentication, request and response mapping, malformed data, error identity,
|
authentication, request and response mapping, malformed data, error identity,
|
||||||
cancellation, and response-body suppression. They use local test servers and
|
cancellation, and response-body suppression. The root transport contract test
|
||||||
test transports; the default suite makes no live or paid provider requests.
|
also verifies that resolved backend settings reach this client without
|
||||||
|
serializing backend identity. All use local test servers or test transports;
|
||||||
|
the default suite makes no live or paid provider requests.
|
||||||
|
|||||||
@@ -11,20 +11,22 @@ contributor workflow and validation.
|
|||||||
|
|
||||||
| Component | Implemented responsibility | References |
|
| Component | Implemented responsibility | References |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| Root `promptkit` package | Provides the supported engine facade, source and injection options, public request and result values, built-in profile construction, extension interfaces, value conversion, redacted formatting, and public error mapping. | [Package GoDoc](../../doc.go), [engine assembly](../../engine.go) |
|
| Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request and result values, profile construction, extension interfaces, value conversion, redacted formatting, and public error mapping. | [Package GoDoc](../../doc.go), [backend API](../../backends.go), [engine assembly](../../engine.go) |
|
||||||
| `examples/go-library/prepare` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, and `Prepare`. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
|
| `examples/go-library/prepare` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, and `Prepare`. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
|
||||||
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
||||||
|
| `internal/backend` | Constructs each engine's immutable registry from the built-in OpenRouter definition and consumer additions, validates and defensively copies definitions through the shared JSON-value package, and consumes the LLM-owned OpenAI-compatible reserved request-field rule. | [Backend registry](../../internal/backend/registry.go) |
|
||||||
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation. | [Domain declarations](../../internal/domain/domain.go) |
|
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation. | [Domain declarations](../../internal/domain/domain.go) |
|
||||||
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
||||||
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
||||||
|
| `internal/jsonvalue` | Validates and deeply copies JSON-compatible extra-parameter trees while preserving supported concrete value types. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
||||||
| `internal/promptdef` | Loads strictly decoded, validated prompt definitions from filesystem and `fs.FS` sources, including version selection and contained file-backed message content. | [Framework formats](../formats.md), [prompt-definition repository](../../internal/promptdef/filesystem_repository.go) |
|
| `internal/promptdef` | Loads strictly decoded, validated prompt definitions from filesystem and `fs.FS` sources, including version selection and contained file-backed message content. | [Framework formats](../formats.md), [prompt-definition repository](../../internal/promptdef/filesystem_repository.go) |
|
||||||
| `internal/profile` | Loads strictly decoded, validated execution profiles from filesystem and `fs.FS` sources and composes repositories with error-preserving fallback. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go) |
|
| `internal/profile` | Loads strictly decoded, validated execution profiles, including backend selection, from filesystem and `fs.FS` sources and composes repositories with error-preserving fallback. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go) |
|
||||||
| `internal/profile/builtin` | Embeds the built-in execution profile catalog and combines it with an optional primary repository. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) |
|
| `internal/profile/builtin` | Embeds the built-in profile catalog, whose entries select OpenRouter, and combines it with an optional primary repository. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) |
|
||||||
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
|
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
|
||||||
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
||||||
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
||||||
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests, response decoding, authentication, and deadline handling. | [Internal model client](llm.md) |
|
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
||||||
| `internal/usecase` | Coordinates preparation and execution across internal sources, rendering, artifact loading, generation, validation, and optional repair. | [Internal runner](runner.md) |
|
| `internal/usecase` | Resolves backend, profile, and request settings and coordinates preparation and execution across internal sources, rendering, artifact loading, generation, validation, and optional repair. | [Internal runner](runner.md) |
|
||||||
|
|
||||||
The root package assembles these internal components without exposing their
|
The root package assembles these internal components without exposing their
|
||||||
representations. Consumers depend only on the root facade.
|
representations. Consumers depend only on the root facade.
|
||||||
|
|||||||
@@ -17,10 +17,12 @@ and override semantics consumed by the runner.
|
|||||||
## Collaborators
|
## Collaborators
|
||||||
|
|
||||||
`Runner` coordinates narrow internal interfaces for prompt definitions,
|
`Runner` coordinates narrow internal interfaces for prompt definitions,
|
||||||
profiles, artifacts, rendering, model generation, and validation. Schema
|
profiles, backend resolution, artifacts, rendering, model generation, and
|
||||||
documents are loaded through the validator's optional schema-loader interface.
|
validation. The root engine supplies one immutable registry containing the
|
||||||
An output repairer can be injected internally, but the ordinary runner
|
built-in backend and validated consumer additions. Schema documents are loaded
|
||||||
constructor does not enable one.
|
through the validator's optional schema-loader interface. An output repairer
|
||||||
|
can be injected internally, but the ordinary runner constructor does not
|
||||||
|
enable one.
|
||||||
|
|
||||||
Each invocation carries its state in request, prepared-run, and result values.
|
Each invocation carries its state in request, prepared-run, and result values.
|
||||||
The runner has no durable run or session store.
|
The runner has no durable run or session store.
|
||||||
@@ -29,23 +31,42 @@ The runner has no durable run or session store.
|
|||||||
|
|
||||||
`Prepare` performs the reusable pre-generation workflow:
|
`Prepare` performs the reusable pre-generation workflow:
|
||||||
|
|
||||||
1. validate the prompt selection and load the prompt definition;
|
1. validate the required prompt selection and normalize any direct session ID;
|
||||||
2. hash the loaded definition;
|
2. load the prompt definition and hash the original definition;
|
||||||
3. select the request profile or the prompt's default profile;
|
3. select the request profile or the prompt's default profile;
|
||||||
4. resolve application-neutral defaults, profile values, and explicit request
|
4. resolve the profile's backend ID, when present;
|
||||||
overrides in that order;
|
5. resolve application-neutral defaults, backend defaults, profile values,
|
||||||
5. validate endpoint, model, numeric overrides, and credential requirements;
|
and explicit request overrides in that order;
|
||||||
6. resolve the output contract and load a structured-output schema when
|
6. validate endpoint, model, numeric overrides, and credential requirements;
|
||||||
|
7. resolve the output contract and load a structured-output schema when
|
||||||
required;
|
required;
|
||||||
7. load and hash input artifacts;
|
8. load and hash input artifacts;
|
||||||
8. render and hash the prompt; and
|
9. render messages, resolve the effective session ID, and hash the effective
|
||||||
9. return the effective settings, source identities, messages, hashes, and
|
rendered prompt; and
|
||||||
|
10. return the effective settings, source identities, messages, hashes, and
|
||||||
preparation timing.
|
preparation timing.
|
||||||
|
|
||||||
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
|
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
|
||||||
out-of-range values fail as invalid requests. A direct API key takes
|
out-of-range values fail as invalid requests. Endpoint overrides do not change
|
||||||
precedence over environment lookup for execution; secret values remain
|
the selected backend identity. Non-empty extra-parameter maps replace whole
|
||||||
excluded from serialized metadata.
|
lower-precedence maps. A direct API key takes precedence over environment
|
||||||
|
lookup; otherwise request, profile, and backend environment-variable names
|
||||||
|
apply in that order. A profile requiring a direct key clears an inherited
|
||||||
|
backend environment name unless the request supplies its own name. Secret
|
||||||
|
values remain excluded from serialized metadata.
|
||||||
|
|
||||||
|
Reasoning overrides are tri-state: nil inherits the profile, a pointer to a
|
||||||
|
nonblank string trims and replaces it, and a pointer to a blank string clears
|
||||||
|
it. A nonblank direct session is normalized before source loading, bypasses
|
||||||
|
the prompt session template, and is applied after ordinary message rendering.
|
||||||
|
A blank direct value retains prompt-template behavior. The runner clears the
|
||||||
|
template only on a value copy of the definition, so the definition hash always
|
||||||
|
describes the original source while the rendered-prompt hash includes the
|
||||||
|
effective direct or rendered session.
|
||||||
|
|
||||||
|
The registry is read-only after engine construction. Concurrent `Prepare` and
|
||||||
|
`Run` calls resolve independent defensive backend values and keep all
|
||||||
|
invocation state local.
|
||||||
|
|
||||||
## Run Flow
|
## Run Flow
|
||||||
|
|
||||||
@@ -56,12 +77,16 @@ an inability to perform validation is an operational error.
|
|||||||
|
|
||||||
When an internal repairer is present, a JSON or JSON Schema content failure can
|
When an internal repairer is present, a JSON or JSON Schema content failure can
|
||||||
trigger bounded repair attempts. Repair receives the effective execution
|
trigger bounded repair attempts. Repair receives the effective execution
|
||||||
target, validation errors, prior output, and structured-output specification.
|
target and session ID, validation errors, prior output, and structured-output
|
||||||
This capability remains internal and is not a public option.
|
specification. This capability remains internal and is not a public option.
|
||||||
|
|
||||||
A successful result includes the output artifact and raw output, validation
|
A successful result includes the output artifact and raw output, validation
|
||||||
state, prompt and rendered-prompt hashes, selected profile, effective settings,
|
state, effective session ID, prompt and rendered-prompt hashes, selected
|
||||||
input hashes, token usage, a generated run identifier, and UTC timing.
|
profile and backend, effective settings, input hashes, token usage, a generated
|
||||||
|
run identifier, and UTC timing. The same effective session reaches initial
|
||||||
|
generation and any repair attempt through the rendered prompt. The same
|
||||||
|
effective target, including backend identity, reaches generation and any
|
||||||
|
repair attempt.
|
||||||
|
|
||||||
## Failure Categories
|
## Failure Categories
|
||||||
|
|
||||||
@@ -71,12 +96,17 @@ validation failures. Wrapping preserves the package identities mapped by the
|
|||||||
public facade and retains collaborator identities where they are part of the
|
public facade and retains collaborator identities where they are part of the
|
||||||
internal contract. Context cancellation propagates through the invoked
|
internal contract. Context cancellation propagates through the invoked
|
||||||
collaborator and is classified by the owning operation.
|
collaborator and is classified by the owning operation.
|
||||||
|
An overlong direct session is an invalid request before source loading, while
|
||||||
|
an invalid or overlong prompt session template remains a prompt-render failure.
|
||||||
|
An unknown selected backend, or a selected backend with no configured resolver,
|
||||||
|
is classified as a profile-load failure.
|
||||||
|
|
||||||
## Test Ownership And Changes
|
## Test Ownership And Changes
|
||||||
|
|
||||||
The [runner tests](../../internal/usecase/runner_test.go) own preparation order,
|
The [runner tests](../../internal/usecase/runner_test.go) own preparation order,
|
||||||
selection and override precedence, schema-before-generation behavior, hashing,
|
selection and override precedence, direct-session resolution,
|
||||||
generation and validation outcomes, bounded repair, credentials and redaction,
|
schema-before-generation behavior, hashing, generation and validation
|
||||||
|
outcomes, backend propagation, bounded repair, credentials and redaction,
|
||||||
error categories, artifact metadata, usage, and timing.
|
error categories, artifact metadata, usage, and timing.
|
||||||
|
|
||||||
Changes to orchestration should continue to use the existing package
|
Changes to orchestration should continue to use the existing package
|
||||||
|
|||||||
@@ -24,13 +24,20 @@ duplicate detection, and source containment:
|
|||||||
|
|
||||||
`internal/profile` loads and validates execution profiles from an
|
`internal/profile` loads and validates execution profiles from an
|
||||||
operating-system filesystem or an `fs.FS`. It supports a primary repository
|
operating-system filesystem or an `fs.FS`. It supports a primary repository
|
||||||
with fallback only when the primary reports that a profile is absent.
|
with fallback only when the primary reports that a profile is absent. Strict
|
||||||
|
YAML decoding recognizes the optional `backend` field, trims its value, and
|
||||||
|
requires a model plus at least one non-blank backend or endpoint. Loading does
|
||||||
|
not check registry membership because the available registry belongs to the
|
||||||
|
assembled engine; the runner checks membership during preparation.
|
||||||
|
|
||||||
`internal/profile/builtin` embeds the maintained built-in profile catalog and
|
`internal/profile/builtin` embeds the maintained built-in profile catalog and
|
||||||
can place a caller-selected repository ahead of that catalog. Profile behavior
|
can place a caller-selected repository ahead of that catalog. Every embedded
|
||||||
is owned by the
|
profile selects `openrouter` and inherits its endpoint and credential
|
||||||
|
environment-variable name from the built-in backend registry rather than
|
||||||
|
repeating those values. Profile behavior is owned by the
|
||||||
[profile repository tests](../../internal/profile/repository_test.go), while
|
[profile repository tests](../../internal/profile/repository_test.go), while
|
||||||
catalog completeness, duplicate IDs, and overlay behavior are owned by the
|
catalog completeness, the backend-selection invariant, duplicate IDs, and
|
||||||
|
overlay behavior are owned by the
|
||||||
[built-in repository tests](../../internal/profile/builtin/repository_test.go).
|
[built-in repository tests](../../internal/profile/builtin/repository_test.go).
|
||||||
|
|
||||||
## Ordinary Artifacts
|
## Ordinary Artifacts
|
||||||
|
|||||||
@@ -21,10 +21,14 @@ The implemented internal components consist of:
|
|||||||
|
|
||||||
- `internal/domain`, which owns framework data values shared by later internal
|
- `internal/domain`, which owns framework data values shared by later internal
|
||||||
components;
|
components;
|
||||||
|
- `internal/backend`, which owns validated immutable OpenAI-compatible backend
|
||||||
|
definitions and the built-in OpenRouter definition;
|
||||||
- `internal/defaults`, which owns application-neutral framework defaults and
|
- `internal/defaults`, which owns application-neutral framework defaults and
|
||||||
constructs the default execution target;
|
constructs the default execution target;
|
||||||
- `internal/filecatalog`, which discovers YAML files and provides source-path
|
- `internal/filecatalog`, which discovers YAML files and provides source-path
|
||||||
helpers for filesystem and `fs.FS` consumers;
|
helpers for filesystem and `fs.FS` consumers;
|
||||||
|
- `internal/jsonvalue`, which validates and defensively copies JSON-compatible
|
||||||
|
extra-parameter trees;
|
||||||
- `internal/promptdef`, which loads and validates prompt definitions from
|
- `internal/promptdef`, which loads and validates prompt definitions from
|
||||||
filesystem and `fs.FS` sources;
|
filesystem and `fs.FS` sources;
|
||||||
- `internal/profile`, which loads, validates, and overlays execution profiles
|
- `internal/profile`, which loads, validates, and overlays execution profiles
|
||||||
@@ -45,17 +49,19 @@ The `examples/go-library/prepare` and `examples/go-library/run` packages are
|
|||||||
maintained downstream consumers of the root facade. They do not expose library
|
maintained downstream consumers of the root facade. They do not expose library
|
||||||
packages or participate in internal assembly.
|
packages or participate in internal assembly.
|
||||||
|
|
||||||
The root facade assembles the internal repositories, renderer, validator,
|
The root facade assembles one immutable backend registry, the internal
|
||||||
outbound client, and use-case runner while translating public values and
|
repositories, renderer, validator, outbound client, and use-case runner while
|
||||||
errors at the library boundary. The defaults and renderer depend on the domain
|
translating public values and errors at the library boundary. The registry
|
||||||
model. Prompt-definition and profile repositories use the domain model, file
|
contains built-ins plus validated engine-scoped consumer additions. The
|
||||||
catalog, and YAML decoder. The built-in profile repository supplies an
|
defaults and renderer depend on the domain model. Prompt-definition and
|
||||||
embedded `fs.FS` to the profile package. Artifact reading uses the domain model
|
profile repositories use the domain model, file catalog, and YAML decoder. The
|
||||||
and application-neutral defaults. Validation uses the domain model, file
|
built-in profile repository supplies an embedded `fs.FS` to the profile
|
||||||
catalog, and JSON Schema implementation. The model client uses the domain
|
package. Artifact reading uses the domain model and application-neutral
|
||||||
model, application-neutral defaults, and an injected or standard-library HTTP
|
defaults. Validation uses the domain model, file catalog, and JSON Schema
|
||||||
client. The use-case runner depends on the narrow interfaces owned by each
|
implementation. The model client uses the domain model, application-neutral
|
||||||
internal component.
|
defaults, and an injected or standard-library HTTP client. The use-case runner
|
||||||
|
depends on the narrow interfaces owned by each internal component, including
|
||||||
|
backend lookup.
|
||||||
|
|
||||||
The current implementation follows this dependency direction:
|
The current implementation follows this dependency direction:
|
||||||
|
|
||||||
@@ -72,9 +78,13 @@ downstream consumers, including Scriptorium
|
|||||||
narrow injected abstractions
|
narrow injected abstractions
|
||||||
```
|
```
|
||||||
|
|
||||||
The facade coordinates internal components and adapts the supported public
|
The backend registry depends on the domain model and shared JSON-value
|
||||||
extension interfaces to narrow internal abstractions. Internal components must
|
validation, has no mutation API after construction, and consumes the
|
||||||
not depend on consumers or on Scriptorium.
|
OpenAI-compatible reserved request-field rule owned by the model client. The
|
||||||
|
model client does not depend on registry configuration. The facade coordinates
|
||||||
|
internal components and adapts
|
||||||
|
the supported public extension interfaces to narrow internal abstractions.
|
||||||
|
Internal components must not depend on consumers or on Scriptorium.
|
||||||
|
|
||||||
## Repository And Consumer Boundary
|
## Repository And Consumer Boundary
|
||||||
|
|
||||||
|
|||||||
236
docs/roadmap/concurrency.md
Normal file
236
docs/roadmap/concurrency.md
Normal file
@@ -0,0 +1,236 @@
|
|||||||
|
# Backend-Specific Concurrency Management
|
||||||
|
|
||||||
|
**Status:** Accepted.
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
This roadmap defines the scope and target end state for engine-local,
|
||||||
|
backend-specific concurrency management. It records the intended capability,
|
||||||
|
consumer value, and important policy choices.
|
||||||
|
|
||||||
|
This document is planning material, not a description of current behavior.
|
||||||
|
Current exported contracts remain owned by Go declarations and GoDoc, backend
|
||||||
|
registration guidance by the
|
||||||
|
[consumer guide](../consumers/pkg-promptkit.md#register-a-custom-backend), and
|
||||||
|
implemented orchestration by the
|
||||||
|
[internal runner document](../internal/runner.md).
|
||||||
|
|
||||||
|
## Motivation
|
||||||
|
|
||||||
|
Different model backends can sustain very different request loads. A local
|
||||||
|
network endpoint may need a small concurrency limit, while OpenRouter can
|
||||||
|
usually accept substantially more simultaneous work. Requiring every consumer
|
||||||
|
to build its own semaphores and queues would duplicate routing knowledge,
|
||||||
|
create inconsistent cancellation behavior, and make it easy for one caller to
|
||||||
|
bypass the intended backend limit.
|
||||||
|
|
||||||
|
Promptkit should own this coordination because it already resolves each run to
|
||||||
|
an engine-scoped backend identity and owns every model-generation call made by
|
||||||
|
the runner. Consumers should continue submitting ready-to-run requests through
|
||||||
|
the synchronous API, including concurrently from multiple goroutines, without
|
||||||
|
implementing their own backend scheduler.
|
||||||
|
|
||||||
|
The buffered queue is a safety boundary, not an ordinary throughput
|
||||||
|
restriction. Its primary purpose is to prevent a bug or unintended submission
|
||||||
|
loop from creating an unbounded in-memory backlog.
|
||||||
|
|
||||||
|
## Scope
|
||||||
|
|
||||||
|
The feature will add optional concurrency policy to registered backends and
|
||||||
|
coordinate `Run` calls against independent per-backend capacity pools.
|
||||||
|
|
||||||
|
Each policy has two distinct controls:
|
||||||
|
|
||||||
|
- an active-generation limit, which protects the backend from too many
|
||||||
|
simultaneous model requests; and
|
||||||
|
- a bounded waiting capacity, which protects the process from admitting an
|
||||||
|
unbounded backlog.
|
||||||
|
|
||||||
|
Concurrency policy belongs to a backend registration. It is not a profile
|
||||||
|
model parameter and cannot be overridden per run. Profiles select the policy
|
||||||
|
through their backend ID, while a profile or request endpoint override remains
|
||||||
|
in the selected backend's pool.
|
||||||
|
|
||||||
|
`Prepare` does not call a model and will remain outside concurrency admission.
|
||||||
|
|
||||||
|
## Defaults And Configuration
|
||||||
|
|
||||||
|
The built-in OpenRouter backend will use:
|
||||||
|
|
||||||
|
- an active-generation limit of 16; and
|
||||||
|
- a waiting capacity of 1024.
|
||||||
|
|
||||||
|
The waiting default is intentionally generous. Reaching it should indicate
|
||||||
|
abnormal submission pressure rather than normal application behavior.
|
||||||
|
|
||||||
|
Consumer-registered backends will remain unlimited unless the consumer
|
||||||
|
configures an active-generation limit. When a consumer enables a limit and
|
||||||
|
does not specify waiting capacity, the waiting capacity will default to 1024.
|
||||||
|
Consumers may configure a different bounded capacity, including zero when
|
||||||
|
they want no admitted backlog beyond the active-limit-sized run set.
|
||||||
|
|
||||||
|
The public representation must distinguish an omitted waiting capacity from
|
||||||
|
an explicit zero.
|
||||||
|
|
||||||
|
Endpoint-only profiles have no backend registration from which to obtain
|
||||||
|
policy and will remain unlimited. A future engine-wide or endpoint-keyed
|
||||||
|
policy can be considered separately if consumers demonstrate that need.
|
||||||
|
|
||||||
|
Invalid limits or capacities will fail engine construction as invalid
|
||||||
|
configuration. Policy values will be copied into engine-owned immutable state
|
||||||
|
along with the rest of the backend registration.
|
||||||
|
|
||||||
|
## Admission And Execution Behavior
|
||||||
|
|
||||||
|
`Run` remains a synchronous, wait-for-result operation. Concurrent callers may
|
||||||
|
block inside `Run` while waiting for their selected backend, then receive the
|
||||||
|
ordinary result or error from that invocation.
|
||||||
|
|
||||||
|
For a configured pool, the active-generation limit plus the waiting capacity
|
||||||
|
defines the maximum number of concurrent `Run` invocations that Promptkit will
|
||||||
|
accept for that backend. A waiting capacity of zero therefore accepts no more
|
||||||
|
runs than the active limit. Admission is immediate: a call either reserves one
|
||||||
|
of those bounded slots or receives the capacity error. An accepted run may
|
||||||
|
then wait internally for active-generation capacity.
|
||||||
|
|
||||||
|
For a limited backend, Promptkit will bound the number of accepted runs before
|
||||||
|
expensive artifact loading, prompt rendering, and large defensive copies where
|
||||||
|
practical. Lightweight prompt, profile, and backend resolution may occur first
|
||||||
|
when it is required to identify the correct capacity pool. This pre-admission
|
||||||
|
resolution must not become a second execution-precedence path with behavior
|
||||||
|
that can drift from `Prepare`.
|
||||||
|
|
||||||
|
An accepted run retains its admission until it completes or fails. Every
|
||||||
|
actual model-generation call for that run must separately observe the
|
||||||
|
backend's active-generation limit. This includes:
|
||||||
|
|
||||||
|
- the initial generation;
|
||||||
|
- every output-repair generation; and
|
||||||
|
- calls made through either the built-in or an injected model client.
|
||||||
|
|
||||||
|
Preparation and output validation should not hold an active-generation permit.
|
||||||
|
A repair remains part of its already-admitted run, but reacquires active
|
||||||
|
generation capacity so repairs cannot exceed the backend limit. It must not be
|
||||||
|
rejected merely because new runs filled the waiting queue after its initial
|
||||||
|
generation.
|
||||||
|
|
||||||
|
Within one backend pool, waiting generation calls should be served in FIFO
|
||||||
|
order, subject to canceled calls being removed. Different backend pools make
|
||||||
|
progress independently; a saturated local backend must not consume
|
||||||
|
OpenRouter's active or waiting capacity.
|
||||||
|
|
||||||
|
The feature will not promise ordering across backend pools or completion order
|
||||||
|
among admitted runs.
|
||||||
|
|
||||||
|
## Capacity Failure And Cancellation
|
||||||
|
|
||||||
|
When a backend's bounded waiting capacity is full, a new `Run` call will fail
|
||||||
|
promptly rather than waiting outside the bounded admission system. The public
|
||||||
|
API will expose a recognizable capacity-exhaustion error identity distinct
|
||||||
|
from invalid configuration, invalid requests, and model-client failures.
|
||||||
|
Rejected calls return no partial result and do not invoke the model client.
|
||||||
|
|
||||||
|
Waiting within the admitted backlog or for active-generation capacity must
|
||||||
|
honor the caller's context. Cancellation or deadline expiry will:
|
||||||
|
|
||||||
|
- stop waiting promptly;
|
||||||
|
- release any admission or generation capacity held by that invocation;
|
||||||
|
- preserve the applicable context error identity; and
|
||||||
|
- avoid invoking the model client if cancellation wins before generation
|
||||||
|
starts.
|
||||||
|
|
||||||
|
Capacity must also be released after preparation, generation, validation,
|
||||||
|
repair, or collaborator failure. One failed or canceled run must not reduce
|
||||||
|
the backend's future usable capacity.
|
||||||
|
|
||||||
|
Elapsed `Run` timing will include time spent waiting after the call is
|
||||||
|
accepted. `PreparedRun` timing will continue to describe preparation rather
|
||||||
|
than queue waiting.
|
||||||
|
|
||||||
|
## Engine And Client Boundaries
|
||||||
|
|
||||||
|
All pools and queued state belong to one `Engine`. Separate engines do not
|
||||||
|
share capacity, even when they register the same backend ID or endpoint. The
|
||||||
|
feature introduces no process-global scheduler.
|
||||||
|
|
||||||
|
The engine will apply policy consistently to the built-in model client and an
|
||||||
|
injected `LLMClient`. Consumers calling their own client outside Promptkit are
|
||||||
|
outside this boundary. Injected clients remain responsible for their internal
|
||||||
|
thread safety and cancellation behavior.
|
||||||
|
|
||||||
|
Backend policy is keyed by the resolved backend ID rather than endpoint text.
|
||||||
|
This preserves stable routing when a selected backend's endpoint is overridden
|
||||||
|
and avoids accidentally combining unrelated registrations that happen to use
|
||||||
|
the same URL.
|
||||||
|
|
||||||
|
## Queue Lifetime And Observability
|
||||||
|
|
||||||
|
Admission state is buffered, ephemeral, and in-process. It is not persisted
|
||||||
|
and has no survival guarantee across engine disposal or process termination.
|
||||||
|
Promptkit will not introduce background job ownership or require consumers to
|
||||||
|
start or stop workers.
|
||||||
|
|
||||||
|
The initial feature does not require public queue-depth metrics, callbacks, or
|
||||||
|
inspection APIs. Capacity errors and ordinary call timing provide the
|
||||||
|
consumer-visible behavior. Operational observability can be added later
|
||||||
|
without coupling the scheduling mechanism to an application logging or
|
||||||
|
metrics system.
|
||||||
|
|
||||||
|
## Compatibility
|
||||||
|
|
||||||
|
Consumer-registered backends and endpoint-only profiles remain unlimited
|
||||||
|
unless concurrency is explicitly configured, preserving their existing
|
||||||
|
behavior.
|
||||||
|
|
||||||
|
The built-in OpenRouter backend will change from unlimited concurrency to a
|
||||||
|
limit of 16 with a bounded waiting capacity of 1024. Ordinary synchronous
|
||||||
|
calls remain unchanged, while unusually high concurrent use may now wait or
|
||||||
|
return the capacity error. This behavioral change must be identified in the
|
||||||
|
release notes for the version that publishes it.
|
||||||
|
|
||||||
|
Adding backend policy fields and a public capacity error is otherwise
|
||||||
|
additive. The change will use a pre-`v1` minor release under Promptkit's
|
||||||
|
[release policy](../release.md#release-model).
|
||||||
|
|
||||||
|
## Non-Goals
|
||||||
|
|
||||||
|
This scope does not include:
|
||||||
|
|
||||||
|
- asynchronous job handles, polling, or detached result delivery;
|
||||||
|
- durable or cross-process queues;
|
||||||
|
- persistence or recovery across engine or process shutdown;
|
||||||
|
- priorities, scheduling weights, or consumer-defined fairness classes;
|
||||||
|
- automatic retries, backoff, rate-limit interpretation, or provider quota
|
||||||
|
discovery;
|
||||||
|
- token-per-minute or request-per-minute rate limiting;
|
||||||
|
- dynamic reconfiguration after engine construction;
|
||||||
|
- per-profile or per-run concurrency overrides;
|
||||||
|
- endpoint-keyed pooling for profiles without a backend ID;
|
||||||
|
- process-global coordination across engines;
|
||||||
|
- application worker lifecycle, logging, tracing, or metrics policy; or
|
||||||
|
- changes to prompt, profile, schema, or model-provider wire formats.
|
||||||
|
|
||||||
|
## Target End State
|
||||||
|
|
||||||
|
This roadmap reaches its target end state when:
|
||||||
|
|
||||||
|
- each engine independently coordinates configured backend capacity;
|
||||||
|
- the built-in OpenRouter backend allows 16 active generations and up to 1024
|
||||||
|
waiting runs;
|
||||||
|
- consumer backends can opt into their own active and waiting limits while
|
||||||
|
remaining unlimited by default;
|
||||||
|
- endpoint overrides retain the selected backend's capacity pool and
|
||||||
|
endpoint-only profiles remain unlimited;
|
||||||
|
- synchronous `Run` callers wait for and receive their ordinary result;
|
||||||
|
- admission is bounded before expensive preparation work where practical;
|
||||||
|
- every initial and repair generation observes the backend's active limit
|
||||||
|
without serializing preparation or validation;
|
||||||
|
- a full waiting queue returns a recognizable capacity error without invoking
|
||||||
|
the model client;
|
||||||
|
- cancellation and all failure paths promptly release capacity and preserve
|
||||||
|
context error identity;
|
||||||
|
- built-in and injected model clients receive the same scheduling behavior;
|
||||||
|
- pools remain ephemeral, engine-scoped, and independent across backend IDs;
|
||||||
|
and
|
||||||
|
- current-state GoDoc, consumer, internal, and release documentation describe
|
||||||
|
the implemented behavior once it lands.
|
||||||
@@ -1,222 +0,0 @@
|
|||||||
# Documentation Hardening Roadmap
|
|
||||||
|
|
||||||
## Purpose
|
|
||||||
|
|
||||||
This roadmap coordinates a focused pass over Promptkit's documentation and
|
|
||||||
public contract. The work should close the gaps identified by the documentation
|
|
||||||
audit, strengthen canonical ownership, and leave consumers and contributors
|
|
||||||
with guidance that is accurate, navigable, and proportionate to their needs.
|
|
||||||
|
|
||||||
This is planning material, not a description of implemented behavior. Follow
|
|
||||||
the [documentation policy](../policy/documentation.md) throughout the work and
|
|
||||||
update current-state documents only when their claims are supported by the
|
|
||||||
implementation and tests.
|
|
||||||
|
|
||||||
## Scope And Principles
|
|
||||||
|
|
||||||
The work covers public GoDoc, consumer guidance, maintained examples, format
|
|
||||||
and integration references, architecture ownership, roadmap lifecycle, and
|
|
||||||
documentation validation.
|
|
||||||
|
|
||||||
- Resolve ambiguous behavior before documenting it as a contract.
|
|
||||||
- Keep exact exported API semantics in Go declarations and GoDoc.
|
|
||||||
- Keep task-oriented guidance in consumer documents and complete runnable
|
|
||||||
artifacts in `examples/`.
|
|
||||||
- Put security-relevant consumer responsibilities at the public boundary, not
|
|
||||||
only in internal contributor documents.
|
|
||||||
- Remove duplicated ownership instead of synchronizing parallel references.
|
|
||||||
- Make documentation checks reproducible where practical.
|
|
||||||
- Preserve existing behavior unless a stage explicitly selects and tests an
|
|
||||||
API change.
|
|
||||||
|
|
||||||
The roadmap does not add new Promptkit features, redesign framework formats,
|
|
||||||
or implement ideas from [the future feature catalog](future.md). If resolving
|
|
||||||
an ambiguity requires a behavioral change, treat that change as a separately
|
|
||||||
reviewable implementation unit and update its canonical documentation in the
|
|
||||||
same unit.
|
|
||||||
|
|
||||||
## Stage 1: Resolve Public Contract Questions
|
|
||||||
|
|
||||||
Before expanding prose, decide the intended contract for exported behavior
|
|
||||||
that is currently ambiguous.
|
|
||||||
|
|
||||||
- [x] Decide whether `RunRequest.Metadata` has a supported observable purpose.
|
|
||||||
Define its propagation and ownership, or remove or deprecate it through an
|
|
||||||
intentional public API change.
|
|
||||||
- [x] Decide and test whether one `Engine` supports concurrent `Prepare` and
|
|
||||||
`Run` calls.
|
|
||||||
- [x] Decide how repeated options of the same category behave, including
|
|
||||||
prompt, profile, schema, model-client, and artifact-reader options.
|
|
||||||
- [x] Define which public values have supported JSON representations.
|
|
||||||
- [x] Define JSON time units and omission behavior, including the relationship
|
|
||||||
between prepared-run and run-result durations.
|
|
||||||
- [x] Decide whether run IDs and exposed hashes have stable formats or must be
|
|
||||||
treated as opaque values.
|
|
||||||
- [x] Confirm the intended transport-timeout default and its zero or negative
|
|
||||||
configuration semantics.
|
|
||||||
- [x] Confirm the supported JSON Schema dialect and reference boundaries,
|
|
||||||
including whether remote references are allowed.
|
|
||||||
|
|
||||||
The selected exported API contracts are implemented and tested. Their durable
|
|
||||||
definitions now belong to the root package declarations and GoDoc.
|
|
||||||
|
|
||||||
One format-level decision remains here until Stage 5 moves it to the framework
|
|
||||||
format reference: JSON Schema uses Draft 2020-12, with that dialect selected
|
|
||||||
when `$schema` is omitted. Same-document fragments and relative references
|
|
||||||
contained by a directory or `fs.FS` schema root are supported. A single-file
|
|
||||||
source supports only references contained in that document. Absolute,
|
|
||||||
escaping, and remote references are rejected.
|
|
||||||
|
|
||||||
**Gate:** Each question has an explicit answer backed by existing behavior or
|
|
||||||
by an accepted implementation change and proportionate tests. No later stage
|
|
||||||
should invent a contract merely to fill a documentation gap.
|
|
||||||
|
|
||||||
## Stage 2: Make GoDoc The Canonical Public Contract
|
|
||||||
|
|
||||||
Strengthen the root package declarations so `go doc` is sufficient to
|
|
||||||
understand exact public behavior without relying on internal documents.
|
|
||||||
|
|
||||||
- [x] Add useful field-level GoDoc to configuration, request, profile,
|
|
||||||
execution-target, result, artifact, validation, structured-output, and model
|
|
||||||
client values.
|
|
||||||
- [x] Document required fields and nil, empty, and zero-value semantics.
|
|
||||||
- [x] Document override, replacement, profile-precedence, and copy-ownership
|
|
||||||
behavior where it belongs to the exported API.
|
|
||||||
- [x] Document credential inputs, redaction, and the values intentionally
|
|
||||||
excluded from serialization.
|
|
||||||
- [x] Give each public error sentinel an accurate comment and document the
|
|
||||||
supported `errors.Is` relationships.
|
|
||||||
- [x] Document engine concurrency and option-composition behavior selected in
|
|
||||||
Stage 1.
|
|
||||||
- [x] Document serialization, time, run-ID, and hash semantics selected in
|
|
||||||
Stage 1.
|
|
||||||
- [x] Review constructor, option, extension-interface, `Prepare`, and `Run`
|
|
||||||
GoDoc for complete failure and cancellation expectations.
|
|
||||||
|
|
||||||
Update the [consumer guide](../consumers/pkg-promptkit.md) to summarize and link
|
|
||||||
to these contracts instead of maintaining exhaustive copies of exported names
|
|
||||||
or exact semantics.
|
|
||||||
|
|
||||||
**Gate:** `go doc -all .` presents a coherent public contract, exported
|
|
||||||
declarations have accurate comments, and contract tests protect every newly
|
|
||||||
documented behavior whose compatibility risk warrants durable coverage.
|
|
||||||
|
|
||||||
## Stage 3: Improve Consumer Safety And Executable Guidance
|
|
||||||
|
|
||||||
Move consumer-relevant security boundaries to the places where consumers will
|
|
||||||
encounter them and add one representative execution workflow.
|
|
||||||
|
|
||||||
- [x] Explain in public GoDoc and the consumer guide that the default file
|
|
||||||
artifact reader accepts unrestricted caller-selected paths.
|
|
||||||
- [x] Make clear that Promptkit does not impose an application root, inbound
|
|
||||||
request-size policy, or untrusted-input security boundary.
|
|
||||||
- [x] Explain that rendered messages, artifact bodies, raw model output, and
|
|
||||||
validation details may be sensitive even when credentials are redacted.
|
|
||||||
- [x] Clarify the responsibilities of injected artifact readers and model
|
|
||||||
clients for cancellation, copying, logging, and secret handling.
|
|
||||||
- [x] Add a maintained offline `Run` example using an injected deterministic
|
|
||||||
model client, without credentials, live network access, or paid calls.
|
|
||||||
- [x] Link the consumer guide to the execution example and keep embedded
|
|
||||||
snippets smaller than the maintained artifact.
|
|
||||||
- [x] Decide whether the existing preparation example should remain separate
|
|
||||||
or share reusable fixtures without obscuring either workflow.
|
|
||||||
|
|
||||||
The preparation and execution examples remain separate, self-contained
|
|
||||||
workflows. Each keeps its own small prompt fixture so consumers can copy or run
|
|
||||||
one example without depending on the other.
|
|
||||||
|
|
||||||
**Gate:** Both preparation and execution have complete, secret-free,
|
|
||||||
deterministic consumer examples, and the consumer guide exposes the important
|
|
||||||
filesystem and data-sensitivity boundaries without leaking internal mechanics.
|
|
||||||
|
|
||||||
## Stage 4: Restore Canonical Ownership
|
|
||||||
|
|
||||||
Remove parallel definitions and make navigation follow the ownership model in
|
|
||||||
the documentation policy.
|
|
||||||
|
|
||||||
- [ ] Reduce the [architecture policy](../policy/architecture.md) to durable
|
|
||||||
boundaries, layers, dependency direction, invariants, and non-goals.
|
|
||||||
- [ ] Keep the exact implemented package and component inventory solely in the
|
|
||||||
[internal component overview](../internal/overview.md).
|
|
||||||
- [ ] Review the consumer guide's public error and option lists so they remain
|
|
||||||
task-oriented summaries rather than duplicate API references.
|
|
||||||
- [ ] Review internal documents for public-contract statements that should be
|
|
||||||
links to GoDoc or the format and integration owners.
|
|
||||||
- [ ] Reconcile the documentation policy's temporary-roadmap lifecycle with
|
|
||||||
the continuing idea-catalog role of `docs/roadmap/future.md`.
|
|
||||||
- [ ] Rephrase or link roadmap statements that depend on exact current API
|
|
||||||
behavior, particularly the runtime reasoning entry.
|
|
||||||
- [ ] Confirm that every document states its audience or purpose and links to
|
|
||||||
the canonical owner of adjacent topics.
|
|
||||||
|
|
||||||
**Gate:** Every authoritative fact has one clear owner, package inventory
|
|
||||||
changes no longer require edits to the architecture policy, and roadmaps
|
|
||||||
cannot be mistaken for current-state references.
|
|
||||||
|
|
||||||
## Stage 5: Refine Format And Integration References
|
|
||||||
|
|
||||||
Close compatibility gaps in the documents that own file formats and outbound
|
|
||||||
wire behavior.
|
|
||||||
|
|
||||||
- [ ] State or canonically link the exact session-ID limit enforced by the
|
|
||||||
OpenAI-compatible client.
|
|
||||||
- [ ] State the configured and default transport-timeout behavior without
|
|
||||||
referring to an unnamed internal default.
|
|
||||||
- [ ] Document the JSON Schema dialect and local, contained, and remote
|
|
||||||
reference behavior selected in Stage 1.
|
|
||||||
- [ ] Clarify structured-output naming and strictness when those values are
|
|
||||||
part of the public or integration contract.
|
|
||||||
- [ ] Add a caveat that built-in profiles are maintained configurations, not a
|
|
||||||
guarantee of continuing third-party model availability.
|
|
||||||
- [ ] Recheck every prompt, profile, schema, credential, request-body, response,
|
|
||||||
timeout, and precedence statement against its owning implementation and
|
|
||||||
tests.
|
|
||||||
|
|
||||||
**Gate:** A consumer can determine the supported file and wire compatibility
|
|
||||||
boundaries without consulting internal source code or relying on unspecified
|
|
||||||
defaults.
|
|
||||||
|
|
||||||
## Stage 6: Make Documentation Validation Reproducible
|
|
||||||
|
|
||||||
Align contributor and release procedures around a small, consistent set of
|
|
||||||
checks.
|
|
||||||
|
|
||||||
- [ ] Use one robust command for checking every tracked Go file with `gofmt`.
|
|
||||||
- [ ] Provide a repository-local or clearly documented command that validates
|
|
||||||
local Markdown targets and heading fragments.
|
|
||||||
- [ ] Decide how published external links are checked without making ordinary
|
|
||||||
validation depend on mutable network services.
|
|
||||||
- [ ] Reconcile the validation descriptions in the
|
|
||||||
[development guide](../development.md),
|
|
||||||
[testing policy](../policy/testing.md), and
|
|
||||||
[release procedure](../release.md) so one document owns each requirement.
|
|
||||||
- [ ] Ensure example validation covers every maintained example added by this
|
|
||||||
roadmap.
|
|
||||||
- [ ] Keep documentation-only validation proportionate while requiring full Go
|
|
||||||
validation when commands, examples, generated output, or checked behavior
|
|
||||||
changes.
|
|
||||||
|
|
||||||
**Gate:** A maintainer can run the documented formatting, link, example, Go,
|
|
||||||
and repository-hygiene checks exactly as written, with no hidden manual
|
|
||||||
procedure for local documentation.
|
|
||||||
|
|
||||||
## Completion Criteria
|
|
||||||
|
|
||||||
The roadmap is complete when:
|
|
||||||
|
|
||||||
- all Stage 1 contract questions are resolved;
|
|
||||||
- GoDoc is the authoritative and sufficient exported API reference;
|
|
||||||
- consumer guidance covers unrestricted file access and sensitive generated
|
|
||||||
data;
|
|
||||||
- maintained offline examples cover both `Prepare` and `Run`;
|
|
||||||
- architecture, inventory, consumer, internal, format, integration, and
|
|
||||||
roadmap documents follow their assigned ownership boundaries;
|
|
||||||
- schema, session, timeout, structured-output, and built-in-profile
|
|
||||||
compatibility statements are explicit;
|
|
||||||
- documentation validation is reproducible and consistent across contributor
|
|
||||||
and release workflows; and
|
|
||||||
- the complete maintainer validation passes.
|
|
||||||
|
|
||||||
After completion, move any durable decisions to GoDoc, policy, format,
|
|
||||||
integration, or ADR owners as appropriate. Remove this roadmap after incoming
|
|
||||||
links are updated; do not retain it as a second current-state reference.
|
|
||||||
@@ -33,78 +33,9 @@ consumers.
|
|||||||
|
|
||||||
## Ideas
|
## Ideas
|
||||||
|
|
||||||
### Extensible LLM backend registry
|
No ideas are currently cataloged. Backend-specific concurrency management has
|
||||||
|
been selected for active planning in the
|
||||||
Introduce a registry that separates backend-specific connection,
|
[focused concurrency roadmap](concurrency.md).
|
||||||
authentication, and limited request defaults from model execution profiles.
|
|
||||||
Initial support would cover OpenAI-compatible backends and include a small
|
|
||||||
built-in catalog, potentially starting with OpenRouter. A profile could select
|
|
||||||
a backend while optionally overriding its default endpoint, and each backend
|
|
||||||
could name an optional environment variable for its API key without storing
|
|
||||||
the credential itself. Downstream consumers could register additional,
|
|
||||||
uniquely named backends, such as OpenAI or unauthenticated local-network
|
|
||||||
services, but could not replace built-in IDs. Model selection and generation
|
|
||||||
settings would remain profile concerns, and custom model clients would remain
|
|
||||||
available for behavior outside the registry's supported protocol.
|
|
||||||
|
|
||||||
### Backend-specific concurrency management
|
|
||||||
|
|
||||||
Extend the proposed LLM backend registry with optional per-backend concurrency
|
|
||||||
limits and bounded, buffered admission queues. Promptkit could then route
|
|
||||||
simultaneous generation requests according to backend capacity while
|
|
||||||
containing accidental runaway submission. Downstream consumers would continue
|
|
||||||
invoking synchronous `Run` calls, including concurrently from multiple
|
|
||||||
goroutines, and each admitted call would wait for and return its ordinary
|
|
||||||
result.
|
|
||||||
|
|
||||||
- Scope limits to an engine instance rather than hidden process-global state.
|
|
||||||
- Give different backend IDs independent capacity pools. A profile endpoint
|
|
||||||
override would remain part of its selected backend's pool.
|
|
||||||
- Configure active concurrency and waiting capacity separately. Concurrency
|
|
||||||
protects the backend, while queue capacity protects the process from
|
|
||||||
admitting an unbounded backlog.
|
|
||||||
- Give queue capacity a generous, configurable bounded default intended as a
|
|
||||||
safety ceiling for bugs or unintended loops rather than a routine
|
|
||||||
application constraint. Select an exact default during implementation
|
|
||||||
planning and measurement.
|
|
||||||
- Reject a call with a recognizable capacity error when its backend queue is
|
|
||||||
full rather than allowing it to wait outside the bounded queue.
|
|
||||||
- Admit requests before expensive preparation and artifact copying where
|
|
||||||
practical so queued work remains lightweight.
|
|
||||||
- Apply a limit to each actual generation request, including repair attempts,
|
|
||||||
without unnecessarily serializing prompt preparation.
|
|
||||||
- Make queued and active waits respect caller cancellation and deadlines.
|
|
||||||
- Treat concurrency as backend policy rather than a profile-level model
|
|
||||||
setting.
|
|
||||||
- Keep the queue ephemeral and in-process, with no survival guarantee across
|
|
||||||
engine or process shutdown.
|
|
||||||
- Preserve the existing execution model as far as practical. Durable jobs,
|
|
||||||
polling, priorities, application worker lifecycle, retries, and
|
|
||||||
cross-process coordination would be separate future capabilities.
|
|
||||||
|
|
||||||
### Explicit per-run session and reasoning controls
|
|
||||||
|
|
||||||
Allow consumers to associate a session ID with each run and to inherit,
|
|
||||||
replace, or explicitly disable the reasoning effort configured by its selected
|
|
||||||
profile. Prompt definitions can currently derive a session ID from a template,
|
|
||||||
and a non-empty runtime `ReasoningEffort` can replace the profile value, but
|
|
||||||
there is no direct request-level session ID and an empty reasoning value means
|
|
||||||
that no override was supplied. These controls would let consumers reuse one
|
|
||||||
prompt and model profile across sessions and reasoning levels without
|
|
||||||
maintaining duplicate definitions.
|
|
||||||
|
|
||||||
- Preserve a prompt's session ID template and a profile's reasoning effort as
|
|
||||||
reusable defaults.
|
|
||||||
- Let a directly supplied per-run session ID take precedence over a rendered
|
|
||||||
prompt default while retaining the existing validation limit and outbound
|
|
||||||
representation.
|
|
||||||
- Distinguish an omitted runtime reasoning choice from an explicit request to
|
|
||||||
disable reasoning.
|
|
||||||
- Ensure disabling reasoning omits the corresponding provider request setting
|
|
||||||
rather than relying on a provider-specific magic value.
|
|
||||||
- Keep the effective session ID and reasoning choice visible in prepared and
|
|
||||||
run metadata and available to injected model clients without introducing
|
|
||||||
additional prompt or profile selection mechanisms.
|
|
||||||
|
|
||||||
## Entry Format
|
## Entry Format
|
||||||
|
|
||||||
|
|||||||
828
docs/roadmap/implementation.md
Normal file
828
docs/roadmap/implementation.md
Normal file
@@ -0,0 +1,828 @@
|
|||||||
|
# Backend-Specific Concurrency Management Implementation Plan
|
||||||
|
|
||||||
|
**Status:** Ready for implementation.
|
||||||
|
|
||||||
|
## Purpose
|
||||||
|
|
||||||
|
This document is the decision-complete implementation plan for
|
||||||
|
[backend-specific concurrency management](concurrency.md). It is written for a
|
||||||
|
coding agent that will implement each stage in order.
|
||||||
|
|
||||||
|
The feature roadmap owns the intended capability, consumer value, policy
|
||||||
|
choices, compatibility decision, and target end state. This document owns the
|
||||||
|
concrete API, internal representation, scheduling architecture, implementation
|
||||||
|
sequence, test ownership, documentation updates, and completion gates.
|
||||||
|
|
||||||
|
## Implementation Rules
|
||||||
|
|
||||||
|
- Complete the stages in order. Keep the repository compiling and the focused
|
||||||
|
tests passing at every stage boundary.
|
||||||
|
- Preserve unrelated working-tree changes. In particular, `concurrency.md` and
|
||||||
|
the removal of its source idea from `future.md` may already be uncommitted
|
||||||
|
when implementation begins; retain both.
|
||||||
|
- Follow every policy under `docs/policy/`, the task-specific reading guide in
|
||||||
|
`docs/development.md`, and the target behavior in `concurrency.md`.
|
||||||
|
- Keep the public API in the root `promptkit` package and implementation
|
||||||
|
details under `internal/`. Do not expose scheduler types or create another
|
||||||
|
public package.
|
||||||
|
- Use only the Go standard library for scheduling. Do not add a queue,
|
||||||
|
semaphore, worker-pool, or metrics dependency.
|
||||||
|
- Preserve synchronous, wait-for-result `Run`, unrestricted `Prepare`,
|
||||||
|
engine-local state, endpoint-only profiles, backend-selected profiles,
|
||||||
|
backend identity through endpoint overrides, and injected `LLMClient`
|
||||||
|
behavior.
|
||||||
|
- Do not broaden the work into asynchronous jobs, durable queues, retries,
|
||||||
|
rate limiting, dynamic configuration, priorities, worker lifecycle,
|
||||||
|
endpoint-keyed pools, or public queue observability.
|
||||||
|
- Keep all tests deterministic, bounded, offline, and race-safe. Coordinate
|
||||||
|
concurrent tests with channels and barriers rather than timing assumptions
|
||||||
|
or live providers.
|
||||||
|
- Update exact GoDoc with each exported declaration change. Update durable
|
||||||
|
current-state documents only after the corresponding behavior is
|
||||||
|
implemented.
|
||||||
|
- Test configurable mechanisms with small test-owned limits. Assert the exact
|
||||||
|
OpenRouter `16` and default queue `1024` values only at the registry contract
|
||||||
|
that owns those operational defaults.
|
||||||
|
- Do not create a release, change a module version, or tag a commit. The final
|
||||||
|
implementation handoff must identify the built-in OpenRouter behavior change
|
||||||
|
for the next pre-`v1` minor release.
|
||||||
|
|
||||||
|
## Fixed Design
|
||||||
|
|
||||||
|
### Public Backend Configuration
|
||||||
|
|
||||||
|
Append these fields to the existing root `Backend` type in `backends.go`:
|
||||||
|
|
||||||
|
```go
|
||||||
|
type Backend struct {
|
||||||
|
// Existing fields remain unchanged and in their current order.
|
||||||
|
|
||||||
|
ConcurrencyLimit int
|
||||||
|
QueueCapacity *int
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Use these exact semantics:
|
||||||
|
|
||||||
|
| Public values | Meaning |
|
||||||
|
| --- | --- |
|
||||||
|
| `ConcurrencyLimit == 0`, `QueueCapacity == nil` | Unlimited backend; preserve current behavior. |
|
||||||
|
| `ConcurrencyLimit > 0`, `QueueCapacity == nil` | Limit active generations and use the default waiting capacity of 1024. |
|
||||||
|
| `ConcurrencyLimit > 0`, `QueueCapacity != nil` | Limit active generations and use the pointed-to capacity exactly, including zero. |
|
||||||
|
| `ConcurrencyLimit < 0` | Invalid engine configuration. |
|
||||||
|
| `QueueCapacity != nil` and `*QueueCapacity < 0` | Invalid engine configuration. |
|
||||||
|
| `ConcurrencyLimit == 0` and `QueueCapacity != nil` | Invalid engine configuration because a queue without an active limit has no defined consumer value. |
|
||||||
|
|
||||||
|
`ConcurrencyLimit` counts simultaneous calls to the engine-owned internal
|
||||||
|
model-client boundary for this backend. `QueueCapacity` controls additional
|
||||||
|
accepted `Run` invocations beyond that limit. The maximum admitted runs for a
|
||||||
|
limited backend is therefore:
|
||||||
|
|
||||||
|
```text
|
||||||
|
ConcurrencyLimit + effective QueueCapacity
|
||||||
|
```
|
||||||
|
|
||||||
|
Guard that addition against integer overflow during backend validation.
|
||||||
|
Do not impose an arbitrary upper bound beyond non-negativity and overflow
|
||||||
|
safety.
|
||||||
|
|
||||||
|
The `QueueCapacity` pointer exists only to distinguish omission from explicit
|
||||||
|
zero. `WithBackend` and `NewEngine` must not retain the caller's pointer.
|
||||||
|
`Backend` continues to have no stable JSON representation, and consumers
|
||||||
|
remain directed to keyed literals.
|
||||||
|
|
||||||
|
Do not add concurrency fields to `Profile`, `ExecutionTarget`,
|
||||||
|
`ExecutionTargetOverride`, `RunRequest`, prompt or profile files, or stable
|
||||||
|
prepared/result JSON.
|
||||||
|
|
||||||
|
### Built-In And Custom Defaults
|
||||||
|
|
||||||
|
The backend registry owns these exact operational defaults:
|
||||||
|
|
||||||
|
```go
|
||||||
|
const (
|
||||||
|
openRouterConcurrencyLimit = 16
|
||||||
|
defaultQueueCapacity = 1024
|
||||||
|
)
|
||||||
|
```
|
||||||
|
|
||||||
|
The built-in `openrouter` definition has a normalized concurrency limit of 16
|
||||||
|
and queue capacity of 1024.
|
||||||
|
|
||||||
|
Consumer registrations remain unlimited when concurrency is omitted. For a
|
||||||
|
consumer backend with a positive limit and omitted queue capacity, normalize
|
||||||
|
the queue capacity to 1024. Preserve an explicitly configured zero.
|
||||||
|
|
||||||
|
Consumers still cannot replace the reserved `openrouter` registration.
|
||||||
|
Endpoint-only profiles have no backend policy and remain unlimited. A selected
|
||||||
|
backend retains its pool when a profile or request overrides only its endpoint.
|
||||||
|
|
||||||
|
### Internal Backend Representation
|
||||||
|
|
||||||
|
Extend `internal/domain.Backend` with scalar policy values and explicit
|
||||||
|
presence rather than retaining a pointer:
|
||||||
|
|
||||||
|
```go
|
||||||
|
type Backend struct {
|
||||||
|
// Existing fields...
|
||||||
|
ConcurrencyLimit int
|
||||||
|
QueueCapacity int
|
||||||
|
QueueCapacitySet bool
|
||||||
|
}
|
||||||
|
|
||||||
|
type BackendCapacityPolicy struct {
|
||||||
|
ConcurrencyLimit int
|
||||||
|
QueueCapacity int
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
`WithBackend` converts the public pointer into `QueueCapacity` plus
|
||||||
|
`QueueCapacitySet`. Registry normalization validates the combinations above,
|
||||||
|
fills the default, and leaves every limited stored backend with
|
||||||
|
`QueueCapacitySet == true`. Unlimited stored backends retain zero values and
|
||||||
|
`QueueCapacitySet == false`.
|
||||||
|
|
||||||
|
Add this internal registry method:
|
||||||
|
|
||||||
|
```go
|
||||||
|
func (r *Registry) CapacityPolicies() map[string]domain.BackendCapacityPolicy
|
||||||
|
```
|
||||||
|
|
||||||
|
It returns a newly allocated map containing only limited backends. Values are
|
||||||
|
scalars, so callers cannot mutate registry state. The built-in OpenRouter
|
||||||
|
policy is included. `GetBackend` continues returning a defensive backend copy,
|
||||||
|
now including normalized scalar capacity metadata.
|
||||||
|
|
||||||
|
Capacity policy is operational registry metadata. Do not merge it into an
|
||||||
|
execution target or expose it to injected model clients.
|
||||||
|
|
||||||
|
### Public Capacity Error
|
||||||
|
|
||||||
|
Add this root sentinel beside the other run errors in `engine.go`:
|
||||||
|
|
||||||
|
```go
|
||||||
|
var ErrCapacityExceeded = errors.New("backend capacity exceeded")
|
||||||
|
```
|
||||||
|
|
||||||
|
Its GoDoc must state that it identifies a `Run` rejected because the selected
|
||||||
|
backend has already admitted `ConcurrencyLimit + QueueCapacity` runs. It is
|
||||||
|
not an invalid request, an LLM/provider rate-limit response, or an
|
||||||
|
`ErrLLMGenerate` failure.
|
||||||
|
|
||||||
|
The internal capacity component owns a corresponding internal
|
||||||
|
`ErrCapacityExceeded`. Add its mapping in `publicErrorFor` before the broader
|
||||||
|
generation and invalid-request cases. The public error must preserve the
|
||||||
|
internal error through wrapping while matching `ErrCapacityExceeded` with
|
||||||
|
`errors.Is`.
|
||||||
|
|
||||||
|
A capacity rejection returns no partial result and must not invoke the
|
||||||
|
artifact reader, renderer, schema loader, validator, or model client. Prompt,
|
||||||
|
profile, and backend loading needed to select the pool may already have
|
||||||
|
occurred.
|
||||||
|
|
||||||
|
### Internal Capacity Component
|
||||||
|
|
||||||
|
Add `internal/capacity` as the single owner of engine-local run admission and
|
||||||
|
active-generation permits.
|
||||||
|
|
||||||
|
Use these package-level boundaries:
|
||||||
|
|
||||||
|
```go
|
||||||
|
var ErrCapacityExceeded error
|
||||||
|
|
||||||
|
type Manager struct {
|
||||||
|
// Private immutable pool map.
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewManager(
|
||||||
|
policies map[string]domain.BackendCapacityPolicy,
|
||||||
|
) (*Manager, error)
|
||||||
|
|
||||||
|
func (m *Manager) Admit(
|
||||||
|
ctx context.Context,
|
||||||
|
backendID string,
|
||||||
|
) (release func(), err error)
|
||||||
|
|
||||||
|
func NewClient(m *Manager, next llm.Client) llm.Client
|
||||||
|
```
|
||||||
|
|
||||||
|
`NewManager` copies the supplied map and creates one independent pool per
|
||||||
|
limited backend. Defensively reject blank IDs, non-positive concurrency
|
||||||
|
limits, negative queue capacities, or total-capacity overflow even though the
|
||||||
|
registry normally supplies normalized values. Construction creates no worker
|
||||||
|
goroutines.
|
||||||
|
|
||||||
|
An absent manager, blank backend ID, or ID absent from the policy map is
|
||||||
|
unlimited:
|
||||||
|
|
||||||
|
- `Admit` succeeds with a non-nil no-op release function; and
|
||||||
|
- the client wrapper calls the next client directly.
|
||||||
|
|
||||||
|
For a limited pool, `Admit` is immediate and context-aware:
|
||||||
|
|
||||||
|
1. return `ctx.Err()` if the context is already done;
|
||||||
|
2. under the pool lock, compare admitted runs with
|
||||||
|
`ConcurrencyLimit + QueueCapacity`;
|
||||||
|
3. return an error matching internal `ErrCapacityExceeded` when full; or
|
||||||
|
4. increment admitted runs and return an idempotent release function.
|
||||||
|
|
||||||
|
The release function decrements admission exactly once, even if accidentally
|
||||||
|
called more than once. It does not release an active-generation permit; those
|
||||||
|
permits have their own lifetime.
|
||||||
|
|
||||||
|
### FIFO Generation Permits
|
||||||
|
|
||||||
|
`NewClient` returns an internal `llm.Client` wrapper around either the built-in
|
||||||
|
client or the public-client adapter. It must preserve requests, successful
|
||||||
|
responses, nil responses, and collaborator error identities exactly.
|
||||||
|
`next` must be non-nil; `NewEngine` and internal runner construction maintain
|
||||||
|
that invariant. A nil manager returns `next` unchanged.
|
||||||
|
|
||||||
|
For a configured backend ID, the wrapper:
|
||||||
|
|
||||||
|
1. acquires one active-generation permit from the matching pool;
|
||||||
|
2. waits in FIFO order when the active count equals `ConcurrencyLimit`;
|
||||||
|
3. removes a canceled waiter and returns `ctx.Err()` when cancellation wins
|
||||||
|
before the permit is granted;
|
||||||
|
4. invokes the next client only after a permit is granted; and
|
||||||
|
5. releases the permit with `defer` after every success, nil response,
|
||||||
|
collaborator error, panic unwinding, or context outcome.
|
||||||
|
|
||||||
|
Implement FIFO and cancellation explicitly with a mutex and an ordered waiter
|
||||||
|
list. A channel used only as a counting semaphore is insufficient because it
|
||||||
|
does not define FIFO ordering or safe removal of canceled waiters.
|
||||||
|
|
||||||
|
Permit grant and cancellation must have one lock-protected linearization
|
||||||
|
point. If cancellation removes the waiter first, do not invoke the next
|
||||||
|
client. If grant wins first, invoke the next client with the caller's context;
|
||||||
|
the next client may then observe cancellation normally. Never lose or
|
||||||
|
double-release a permit in this race.
|
||||||
|
|
||||||
|
Releasing a permit transfers it to the oldest non-canceled waiter before
|
||||||
|
making it generally available. Different backend pools never share admission
|
||||||
|
or active counts.
|
||||||
|
|
||||||
|
The active wrapper enforces its limit even if an internal caller invokes it
|
||||||
|
without a run admission lease. Bounded backlog is guaranteed for ordinary
|
||||||
|
engine `Run` calls by the runner admission path; no public API exposes the
|
||||||
|
wrapped internal client directly.
|
||||||
|
|
||||||
|
### Engine Assembly
|
||||||
|
|
||||||
|
In `NewEngine`, after constructing the validated backend registry:
|
||||||
|
|
||||||
|
1. obtain `backendRegistry.CapacityPolicies()`;
|
||||||
|
2. construct one `capacity.Manager`;
|
||||||
|
3. construct the selected base internal LLM client exactly as today;
|
||||||
|
4. wrap that base client with `capacity.NewClient`; and
|
||||||
|
5. pass both the wrapped client and manager-as-admitter to the runner.
|
||||||
|
|
||||||
|
Every `NewEngine` call constructs a distinct manager. Do not cache managers,
|
||||||
|
pools, or policies in package globals. The wrapper must be applied after a
|
||||||
|
public injected client is adapted to `internal/llm.Client`, so built-in and
|
||||||
|
injected clients receive identical scheduling behavior.
|
||||||
|
|
||||||
|
If `NewManager` reports a defensive configuration error, make `NewEngine`
|
||||||
|
return an error matching `ErrInvalidConfig`.
|
||||||
|
|
||||||
|
`Prepare` does not use the manager. An injected client remains required to be
|
||||||
|
safe for concurrent calls because different backend pools and unlimited
|
||||||
|
backends may still invoke it concurrently.
|
||||||
|
|
||||||
|
### Shared Two-Phase Preparation
|
||||||
|
|
||||||
|
Refactor `internal/usecase.Runner` so `Prepare` and `Run` share one preparation
|
||||||
|
pipeline with two private phases. Do not duplicate prompt/profile/backend
|
||||||
|
selection or execution precedence.
|
||||||
|
|
||||||
|
The first phase resolves only the state required before admission:
|
||||||
|
|
||||||
|
1. validate `PromptID`;
|
||||||
|
2. normalize the direct session ID;
|
||||||
|
3. load the prompt definition;
|
||||||
|
4. hash the original prompt definition at its existing error-order position;
|
||||||
|
5. select and load the execution profile;
|
||||||
|
6. resolve the selected backend;
|
||||||
|
7. resolve and validate the effective execution target and credentials; and
|
||||||
|
8. resolve the effective output contract without loading its schema.
|
||||||
|
|
||||||
|
Return a private state value containing the loaded definition, normalized
|
||||||
|
direct session, prompt-definition hash, selected profile ID, effective target,
|
||||||
|
numeric-presence metadata, effective output contract, and preparation start
|
||||||
|
time. Keep this value private to `internal/usecase`.
|
||||||
|
|
||||||
|
The second phase consumes that state and performs:
|
||||||
|
|
||||||
|
1. structured-output schema loading;
|
||||||
|
2. artifact loading and input hashing;
|
||||||
|
3. message and prompt-session rendering;
|
||||||
|
4. direct-session application;
|
||||||
|
5. rendered-prompt hashing; and
|
||||||
|
6. `PreparedRun` construction and timing.
|
||||||
|
|
||||||
|
Preserve every existing precedence rule, error identity, direct-session
|
||||||
|
template bypass, hash input, selected identity, copy guarantee, and timing
|
||||||
|
field. Do not reload the prompt, profile, or backend between phases.
|
||||||
|
|
||||||
|
`Runner.Prepare` records its start time, runs both phases consecutively, and
|
||||||
|
never calls admission. Its behavior and error ordering remain unchanged.
|
||||||
|
|
||||||
|
`Runner.Run` records its existing run start time, runs the first preparation
|
||||||
|
phase, and then calls:
|
||||||
|
|
||||||
|
```go
|
||||||
|
release, err := r.admitter.Admit(ctx, effectiveBackendID)
|
||||||
|
```
|
||||||
|
|
||||||
|
Use a narrow use-case-owned interface with the same signature:
|
||||||
|
|
||||||
|
```go
|
||||||
|
type RunAdmitter interface {
|
||||||
|
Admit(context.Context, string) (func(), error)
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
A nil admitter means unlimited behavior for internal constructors and tests.
|
||||||
|
On successful admission, immediately `defer release()` around the remainder of
|
||||||
|
the run. Then run the second preparation phase, initial generation,
|
||||||
|
validation, and all repair attempts.
|
||||||
|
|
||||||
|
If admission returns internal `capacity.ErrCapacityExceeded`, add useful
|
||||||
|
backend context without changing its identity. If it returns `ctx.Err()`,
|
||||||
|
preserve that identity directly rather than recategorizing it as invalid
|
||||||
|
request or generation failure.
|
||||||
|
|
||||||
|
This refactor intentionally replaces the current literal `Run`-calls-`Prepare`
|
||||||
|
implementation with shared private phases. Update current-state documentation
|
||||||
|
to describe one shared pipeline rather than retaining an inaccurate call-graph
|
||||||
|
claim.
|
||||||
|
|
||||||
|
### Generation And Repair Lifetime
|
||||||
|
|
||||||
|
The admission lease covers the entire accepted run:
|
||||||
|
|
||||||
|
- second-phase preparation;
|
||||||
|
- initial generation;
|
||||||
|
- validation;
|
||||||
|
- every repair; and
|
||||||
|
- all failure and cancellation exits.
|
||||||
|
|
||||||
|
Preparation and validation do not hold an active-generation permit. The
|
||||||
|
wrapped client acquires a permit only around each actual `Generate` call.
|
||||||
|
|
||||||
|
The runner's initial generation already carries the effective backend ID in
|
||||||
|
`GenerateRequest.Target`. Preserve that value. `RepairRequest.Target` and the
|
||||||
|
default repairer's generated request must continue carrying the same backend
|
||||||
|
ID, allowing each repair to reacquire the same pool's active permit.
|
||||||
|
|
||||||
|
When testing or constructing `NewRunnerWithRepairer`, pass the same wrapped
|
||||||
|
client to both the runner and `NewDefaultOutputRepairer`. Do not add capacity
|
||||||
|
state to `RepairRequest`, `ExecutionTarget`, or public generation values.
|
||||||
|
|
||||||
|
A repair remains within its existing admission lease. It waits for a FIFO
|
||||||
|
active permit but never performs a second bounded admission and therefore
|
||||||
|
cannot fail merely because later runs filled the admission capacity.
|
||||||
|
|
||||||
|
### Error And Cancellation Semantics
|
||||||
|
|
||||||
|
The required public outcomes are:
|
||||||
|
|
||||||
|
| Situation | Required error identity |
|
||||||
|
| --- | --- |
|
||||||
|
| Admission capacity is full | `ErrCapacityExceeded` only; not `ErrInvalidRequest` or `ErrLLMGenerate`. |
|
||||||
|
| Context is done before admission succeeds | Preserve `ctx.Err()`; do not return capacity exhaustion. |
|
||||||
|
| Context cancels while waiting for an active permit | Preserve `ctx.Err()` through the existing `ErrLLMGenerate` generation category. |
|
||||||
|
| Wrapped client fails after permit acquisition | Preserve existing `ErrLLMGenerate` and collaborator identities. |
|
||||||
|
| Preparation or validation fails after admission | Preserve its existing category and release admission. |
|
||||||
|
|
||||||
|
Maintain the existing rule that `Run` returns no partial result on any
|
||||||
|
operational error. Do not add queue status to errors or results.
|
||||||
|
|
||||||
|
`RunResult.Duration` continues to start at runner entry and therefore includes
|
||||||
|
pre-admission resolution, accepted preparation, and active-permit waiting.
|
||||||
|
`PreparedRun.DurationMS` continues to cover only its shared preparation phases;
|
||||||
|
it does not include later generation waiting. Capacity-rejected calls have no
|
||||||
|
result or timing value.
|
||||||
|
|
||||||
|
### Ownership And Concurrency Safety
|
||||||
|
|
||||||
|
The registry, capacity policy map, pool map, and per-pool limits are immutable
|
||||||
|
after engine construction. Only admission counts, active counts, and waiter
|
||||||
|
lists are mutable and must be protected by the owning pool mutex.
|
||||||
|
|
||||||
|
Do not retain public queue pointers, caller request values, contexts, or
|
||||||
|
generation requests after their call completes. A canceled waiter must be
|
||||||
|
unlinked so its context and request cannot remain reachable from the pool.
|
||||||
|
|
||||||
|
Do not hold a pool mutex while:
|
||||||
|
|
||||||
|
- loading or rendering prompts;
|
||||||
|
- reading artifacts or schemas;
|
||||||
|
- invoking a model client;
|
||||||
|
- validating output;
|
||||||
|
- closing a waiter notification channel if the implementation could re-enter
|
||||||
|
pool code; or
|
||||||
|
- calling consumer code.
|
||||||
|
|
||||||
|
No scheduler operation may spawn a goroutine whose lifetime outlasts the
|
||||||
|
calling `Run`. The zero steady-state goroutine count is part of the
|
||||||
|
in-process/no-worker-lifecycle design.
|
||||||
|
|
||||||
|
## Test Ownership
|
||||||
|
|
||||||
|
Use this ownership split and avoid repeating the full policy matrix at every
|
||||||
|
layer:
|
||||||
|
|
||||||
|
- `internal/backend/registry_test.go` owns normalization, validation, the exact
|
||||||
|
OpenRouter policy, the custom default queue, explicit zero, unlimited
|
||||||
|
omission, and policy-map copying.
|
||||||
|
- `internal/capacity/manager_test.go` owns admission bounds, idempotent release,
|
||||||
|
FIFO active permits, cancellation races, capacity recovery, independent
|
||||||
|
pools, unlimited IDs, and observed peak concurrency.
|
||||||
|
- `internal/capacity/client_test.go` owns wrapper request/response/error
|
||||||
|
transparency and the rule that cancellation before grant does not invoke the
|
||||||
|
next client. Combine these with manager tests if one coherent package test
|
||||||
|
expresses the behavior more clearly.
|
||||||
|
- `internal/usecase/runner_test.go` owns two-phase preparation parity, pool
|
||||||
|
selection, admission before expensive work, admission release across run
|
||||||
|
exits, `Prepare` bypass, and repair reuse of the admitted backend.
|
||||||
|
- Root external-package tests own public configuration conversion, assembled
|
||||||
|
engine-local behavior, endpoint-override routing, injected-client limiting,
|
||||||
|
and public capacity/context error identities.
|
||||||
|
- Existing model-client HTTP tests remain unchanged because scheduling does
|
||||||
|
not alter the OpenAI-compatible wire contract.
|
||||||
|
|
||||||
|
Concurrency tests must use test-owned limits such as one or two and
|
||||||
|
channel-controlled blocking clients. Record observed active and peak counts
|
||||||
|
under a mutex or atomics. Do not use `time.Sleep` to infer queue state.
|
||||||
|
Package-internal tests may inspect a waiter list under its mutex through a
|
||||||
|
small test helper when necessary to establish deterministic FIFO ordering; do
|
||||||
|
not add production metrics or hooks solely for tests.
|
||||||
|
|
||||||
|
Do not add separate tests for trivial scalar copies when registry or assembled
|
||||||
|
behavior already protects them.
|
||||||
|
|
||||||
|
## Stage 1 — Backend Policy And Public Configuration
|
||||||
|
|
||||||
|
**Status:** Pending.
|
||||||
|
|
||||||
|
### Goal
|
||||||
|
|
||||||
|
Add the public and internal backend policy representation, normalize all
|
||||||
|
configured states, and expose immutable normalized policies without changing
|
||||||
|
runtime scheduling yet.
|
||||||
|
|
||||||
|
### Work
|
||||||
|
|
||||||
|
1. Add `ConcurrencyLimit` and `QueueCapacity` to `Backend` in `backends.go`
|
||||||
|
with exact GoDoc for unlimited, defaulted, explicit-zero, invalid, and
|
||||||
|
engine-scoped behavior.
|
||||||
|
2. Convert the public queue pointer into scalar value plus presence in
|
||||||
|
`WithBackend`; do not retain the pointer.
|
||||||
|
3. Add the internal backend policy fields and
|
||||||
|
`BackendCapacityPolicy` to `internal/domain/domain.go`.
|
||||||
|
4. Add the two registry-owned constants and configure the built-in OpenRouter
|
||||||
|
definition with 16 and 1024.
|
||||||
|
5. Extend `normalizeBackend` with the fixed validation, defaulting, explicit
|
||||||
|
zero, and overflow rules.
|
||||||
|
6. Add `Registry.CapacityPolicies`, returning only limited policies in a fresh
|
||||||
|
map.
|
||||||
|
7. Update existing backend composite literals and assertions only where the
|
||||||
|
new fields are relevant. Continue using keyed literals.
|
||||||
|
|
||||||
|
### Tests
|
||||||
|
|
||||||
|
1. Extend the exact built-in registry test with the OpenRouter limit and queue.
|
||||||
|
2. Add one coherent table covering unlimited omission, default queue,
|
||||||
|
explicit-zero queue, negative values, queue-without-limit, and total
|
||||||
|
overflow.
|
||||||
|
3. Extend the registry copy/normalization test to prove returned policy maps
|
||||||
|
cannot mutate registry state.
|
||||||
|
4. Add root coverage only if needed to prove the public pointer/presence
|
||||||
|
conversion; do not reproduce registry validation cases at the facade.
|
||||||
|
|
||||||
|
### Focused Validation
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
gofmt -w backends.go internal/domain/domain.go \
|
||||||
|
internal/backend/registry.go internal/backend/registry_test.go
|
||||||
|
go test . ./internal/backend
|
||||||
|
go vet . ./internal/backend
|
||||||
|
git diff --check
|
||||||
|
```
|
||||||
|
|
||||||
|
Include another touched Go test file in `gofmt` only if it actually changed.
|
||||||
|
|
||||||
|
### Completion Gate
|
||||||
|
|
||||||
|
This stage is complete when every public configuration state has one normalized
|
||||||
|
internal meaning, OpenRouter exposes exactly 16/1024, custom backends remain
|
||||||
|
unlimited by omission, and no runtime call is scheduled yet.
|
||||||
|
|
||||||
|
## Stage 2 — Engine-Local Capacity Manager
|
||||||
|
|
||||||
|
**Status:** Pending.
|
||||||
|
|
||||||
|
### Goal
|
||||||
|
|
||||||
|
Implement and prove the bounded admission mechanism and FIFO active-generation
|
||||||
|
client wrapper independently of runner orchestration.
|
||||||
|
|
||||||
|
### Work
|
||||||
|
|
||||||
|
1. Add `internal/capacity/manager.go` with the manager, immutable policy copy,
|
||||||
|
per-backend pools, internal error, immediate admission, idempotent release,
|
||||||
|
and FIFO context-aware active permits.
|
||||||
|
2. Add `internal/capacity/client.go` with the transparent `llm.Client` wrapper.
|
||||||
|
3. Use mutex-protected waiter state and an ordered list; explicitly resolve
|
||||||
|
grant-versus-cancel races.
|
||||||
|
4. Ensure unlimited and independent-pool fast paths avoid queue allocation.
|
||||||
|
5. Do not start workers, timers, cleanup goroutines, or process-global state.
|
||||||
|
|
||||||
|
### Tests
|
||||||
|
|
||||||
|
1. Add a compact constructor-validation table for blank IDs, non-positive
|
||||||
|
limits, negative queues, and total-capacity overflow.
|
||||||
|
2. With a small configured policy, prove that exactly
|
||||||
|
`limit + queueCapacity` admissions succeed, the next matches
|
||||||
|
`ErrCapacityExceeded`, and a release permits another admission.
|
||||||
|
3. Prove release is idempotent.
|
||||||
|
4. Drive more blocked client calls than the active limit and assert observed
|
||||||
|
peak concurrency never exceeds that limit.
|
||||||
|
5. Prove FIFO order with deterministic queue-entry synchronization.
|
||||||
|
6. Cancel the first and a middle waiter and prove they are removed, never call
|
||||||
|
the wrapped client, and do not block later waiters.
|
||||||
|
7. Exercise the grant/cancel race repeatedly under `go test -race`, asserting
|
||||||
|
no permit leak or double invocation.
|
||||||
|
8. Prove different backend IDs proceed independently and blank, unknown, or
|
||||||
|
nil-manager paths remain unlimited.
|
||||||
|
9. Prove request values, successful and nil responses, and collaborator errors
|
||||||
|
pass through unchanged after permit acquisition.
|
||||||
|
|
||||||
|
### Focused Validation
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
gofmt -w internal/capacity/manager.go \
|
||||||
|
internal/capacity/manager_test.go \
|
||||||
|
internal/capacity/client.go \
|
||||||
|
internal/capacity/client_test.go
|
||||||
|
go test ./internal/capacity
|
||||||
|
go test -race ./internal/capacity
|
||||||
|
go vet ./internal/capacity
|
||||||
|
git diff --check
|
||||||
|
```
|
||||||
|
|
||||||
|
If tests are combined into one file, omit the nonexistent file from `gofmt`.
|
||||||
|
|
||||||
|
### Completion Gate
|
||||||
|
|
||||||
|
This stage is complete when the standalone component enforces relational
|
||||||
|
admission and active limits, FIFO cancellation is race-safe, separate pools
|
||||||
|
are independent, and the wrapper is transparent apart from waiting.
|
||||||
|
|
||||||
|
## Stage 3 — Shared Preparation And Early Run Admission
|
||||||
|
|
||||||
|
**Status:** Pending.
|
||||||
|
|
||||||
|
### Goal
|
||||||
|
|
||||||
|
Refactor runner preparation into one shared two-phase pipeline and place
|
||||||
|
bounded admission after backend resolution but before schema, artifact, and
|
||||||
|
rendering work.
|
||||||
|
|
||||||
|
### Work
|
||||||
|
|
||||||
|
1. Add the private pre-admission preparation state and split the existing
|
||||||
|
`Prepare` logic according to the fixed design.
|
||||||
|
2. Make `Runner.Prepare` call both phases without an admitter.
|
||||||
|
3. Add the `RunAdmitter` interface and runner field.
|
||||||
|
4. Update `NewRunner` and `NewRunnerWithRepairer` to accept the optional
|
||||||
|
admitter; update internal call sites with nil until root assembly is wired.
|
||||||
|
5. Change `Runner.Run` to use the first phase, admit by effective backend ID,
|
||||||
|
defer the returned release, and then use the second phase.
|
||||||
|
6. Preserve all existing error precedence, target resolution, hashes,
|
||||||
|
metadata, session behavior, and timing.
|
||||||
|
7. Return capacity and context errors with the fixed identities. Do not invoke
|
||||||
|
later collaborators after rejection.
|
||||||
|
|
||||||
|
### Tests
|
||||||
|
|
||||||
|
1. Keep the existing `Run`/`Prepare` parity coverage passing to prove the
|
||||||
|
shared phases do not drift.
|
||||||
|
2. Add a fake admitter that records backend IDs and release calls.
|
||||||
|
3. Prove a backend-selected run admits with the selected ID even when the
|
||||||
|
endpoint is overridden.
|
||||||
|
4. Prove an endpoint-only run uses the unlimited/blank identity and that
|
||||||
|
`Prepare` never calls admission.
|
||||||
|
5. Reject admission and assert schema, artifact, renderer, validator, repairer,
|
||||||
|
and LLM collaborators are not invoked.
|
||||||
|
6. Prove admission is released after one successful run and representative
|
||||||
|
second-phase, generation, and validation errors. Prefer a small table around
|
||||||
|
the single `defer` invariant rather than duplicating every error test.
|
||||||
|
7. Retain direct-session, backend precedence, credential, hashing, and repair
|
||||||
|
tests unchanged except for constructor arguments.
|
||||||
|
|
||||||
|
### Focused Validation
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
gofmt -w internal/usecase/runner.go \
|
||||||
|
internal/usecase/runner_test.go
|
||||||
|
go test ./internal/usecase
|
||||||
|
go test -race ./internal/usecase
|
||||||
|
go vet ./internal/usecase
|
||||||
|
git diff --check
|
||||||
|
```
|
||||||
|
|
||||||
|
### Completion Gate
|
||||||
|
|
||||||
|
This stage is complete when `Prepare` remains unrestricted, `Run` admits after
|
||||||
|
one canonical routing phase and before expensive completion work, every exit
|
||||||
|
releases admission, and existing preparation semantics remain unchanged.
|
||||||
|
|
||||||
|
## Stage 4 — Engine Assembly And Public Runtime Contract
|
||||||
|
|
||||||
|
**Status:** Pending.
|
||||||
|
|
||||||
|
### Goal
|
||||||
|
|
||||||
|
Wire one manager into each engine, schedule built-in and injected clients,
|
||||||
|
expose the capacity error, and prove assembled runtime behavior.
|
||||||
|
|
||||||
|
### Work
|
||||||
|
|
||||||
|
1. Add public `ErrCapacityExceeded` and its exact GoDoc in `engine.go`.
|
||||||
|
2. Map internal capacity exhaustion in `errors.go`.
|
||||||
|
3. Construct the manager from the registry policy snapshot in `NewEngine`.
|
||||||
|
4. Wrap the selected internal client after built-in or injected-client
|
||||||
|
selection and pass the manager and wrapped client to the runner.
|
||||||
|
5. Update `Engine`, `NewEngine`, `Run`, `WithLLMClient`, and `LLMClient` GoDoc
|
||||||
|
only where concurrency, capacity, or cancellation statements change.
|
||||||
|
6. Ensure manager-construction errors match `ErrInvalidConfig`.
|
||||||
|
7. For internal repair coverage, construct the default repairer with the same
|
||||||
|
wrapped client used by its runner and confirm repair target backend identity
|
||||||
|
remains intact.
|
||||||
|
|
||||||
|
### Tests
|
||||||
|
|
||||||
|
1. Add an external-package assembled test with a small custom limit and a
|
||||||
|
blocking injected client; assert peak generation equals or remains below
|
||||||
|
the configured limit.
|
||||||
|
2. With queue capacity zero, block one accepted run before generation and
|
||||||
|
assert the next matching-backend run returns `ErrCapacityExceeded`, does not
|
||||||
|
match `ErrInvalidRequest` or `ErrLLMGenerate`, returns no result, and never
|
||||||
|
reaches expensive collaborators or the client.
|
||||||
|
3. In the same or another focused workflow, prove an endpoint override remains
|
||||||
|
in the selected backend's pool.
|
||||||
|
4. Prove two engines with the same backend ID have independent capacity.
|
||||||
|
5. Prove an unlimited custom backend and an endpoint-only profile preserve
|
||||||
|
concurrent behavior.
|
||||||
|
6. Cancel a call waiting for an active permit; assert it matches both
|
||||||
|
`context.Canceled` and `ErrLLMGenerate`, never invokes the injected client,
|
||||||
|
and leaves capacity reusable.
|
||||||
|
7. Add one internal repair workflow with concurrent runs or controlled permits
|
||||||
|
showing initial and repair generations never exceed the same backend limit
|
||||||
|
and repairs do not perform a second admission.
|
||||||
|
8. Extend the public error sentinel contract test with
|
||||||
|
`ErrCapacityExceeded`.
|
||||||
|
|
||||||
|
Avoid a second HTTP-level concurrency suite: the capacity client tests and one
|
||||||
|
assembled injected-client workflow already protect the shared wrapper used by
|
||||||
|
the built-in client.
|
||||||
|
|
||||||
|
### Focused Validation
|
||||||
|
|
||||||
|
Run:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
gofmt -w engine.go errors.go backends.go \
|
||||||
|
internal/usecase/runner.go internal/usecase/runner_test.go \
|
||||||
|
engine_test.go public_contract_test.go
|
||||||
|
go test . ./internal/backend ./internal/capacity ./internal/usecase
|
||||||
|
go test -race . ./internal/capacity ./internal/usecase
|
||||||
|
go vet . ./internal/backend ./internal/capacity ./internal/usecase
|
||||||
|
git diff --check
|
||||||
|
```
|
||||||
|
|
||||||
|
Add any newly created capacity files to `gofmt` when they changed in this
|
||||||
|
stage.
|
||||||
|
|
||||||
|
### Completion Gate
|
||||||
|
|
||||||
|
This stage is complete when every engine has independent pools, limited runs
|
||||||
|
are bounded and FIFO at generation, endpoint routing is correct, capacity and
|
||||||
|
context errors are stable, repairs reuse admission, and both client kinds pass
|
||||||
|
through the same wrapper.
|
||||||
|
|
||||||
|
## Stage 5 — Durable Documentation And Final Validation
|
||||||
|
|
||||||
|
**Status:** Pending.
|
||||||
|
|
||||||
|
### Goal
|
||||||
|
|
||||||
|
Move implemented contracts into their durable owners, record compatibility
|
||||||
|
impact, and validate the complete repository.
|
||||||
|
|
||||||
|
### Work
|
||||||
|
|
||||||
|
1. Review every changed exported declaration. Ensure GoDoc is the canonical
|
||||||
|
owner of exact field types, nil/zero semantics, defaulting, error identity,
|
||||||
|
engine scope, concurrency safety, cancellation, and source compatibility.
|
||||||
|
2. Update `doc.go` so its concurrency summary acknowledges backend scheduling
|
||||||
|
while continuing to require injected collaborators to be concurrency-safe.
|
||||||
|
3. Update `docs/consumers/pkg-promptkit.md` with task-oriented examples for:
|
||||||
|
- a limited local backend;
|
||||||
|
- omitted queue capacity selecting 1024;
|
||||||
|
- explicit zero queue capacity; and
|
||||||
|
- handling `ErrCapacityExceeded`.
|
||||||
|
Keep exact field semantics in GoDoc rather than duplicating a full table.
|
||||||
|
4. Add `docs/internal/capacity.md` as the durable owner of pool lifecycle,
|
||||||
|
admission, FIFO active permits, cancellation, client wrapping, and test
|
||||||
|
ownership.
|
||||||
|
5. Add `internal/capacity` to `docs/internal/overview.md`.
|
||||||
|
6. Update `docs/policy/architecture.md` to include the implemented component
|
||||||
|
and root assembly dependency without turning policy into an API reference.
|
||||||
|
7. Update `docs/internal/runner.md` to describe the shared two-phase
|
||||||
|
preparation pipeline, early bounded admission, lease lifetime, generation
|
||||||
|
permits, repairs, capacity failures, and cancellation.
|
||||||
|
8. Review `docs/formats.md`; add only a concise link or clarification if needed
|
||||||
|
to explain that endpoint overrides preserve backend capacity identity.
|
||||||
|
Do not add concurrency fields to YAML.
|
||||||
|
9. Do not change the OpenAI-compatible integration contract or
|
||||||
|
`docs/internal/llm.md` unless implementation changes their current
|
||||||
|
statements; scheduling is outside the provider wire contract and concrete
|
||||||
|
model-client implementation.
|
||||||
|
10. Record in the implementation handoff that built-in OpenRouter now limits
|
||||||
|
active generations to 16 with queue capacity 1024 and that the release
|
||||||
|
must be a pre-`v1` minor release. Do not edit the release procedure or
|
||||||
|
create a tag.
|
||||||
|
11. After every check passes, set `concurrency.md`, this implementation plan,
|
||||||
|
and each stage status to `Complete`. Do not remove the roadmaps in the
|
||||||
|
implementation change; lifecycle retirement follows review.
|
||||||
|
|
||||||
|
### Full Validation
|
||||||
|
|
||||||
|
Run the complete sequence from `docs/development.md`:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
go test ./...
|
||||||
|
go test -race ./...
|
||||||
|
go vet ./...
|
||||||
|
go build ./...
|
||||||
|
go run ./examples/go-library/prepare
|
||||||
|
gofmt -l $(git ls-files '*.go')
|
||||||
|
git diff --check
|
||||||
|
```
|
||||||
|
|
||||||
|
The formatting command must produce no paths. Follow every added or changed
|
||||||
|
Markdown link and confirm its target and heading exist.
|
||||||
|
|
||||||
|
Also inspect:
|
||||||
|
|
||||||
|
```sh
|
||||||
|
git status --short
|
||||||
|
git diff --stat
|
||||||
|
git diff
|
||||||
|
```
|
||||||
|
|
||||||
|
Confirm that:
|
||||||
|
|
||||||
|
- only intended backend, capacity, runner, facade, test, documentation, and
|
||||||
|
roadmap files changed;
|
||||||
|
- no `go.work`, `go.work.sum`, local module replacement, credential,
|
||||||
|
generated binary, coverage output, or unrelated change was introduced;
|
||||||
|
- the built-in OpenRouter policy is exactly 16/1024;
|
||||||
|
- custom and endpoint-only backends remain unlimited by omission;
|
||||||
|
- explicit queue zero is distinguishable from omission;
|
||||||
|
- no capacity value enters execution targets, generated requests, stable JSON,
|
||||||
|
prompt/profile YAML, or provider payloads;
|
||||||
|
- every engine owns distinct pools with no package-global mutable state;
|
||||||
|
- every initial and repair generation uses the active permit wrapper;
|
||||||
|
- capacity and waiter state is released on success, error, panic unwinding,
|
||||||
|
and cancellation;
|
||||||
|
- concurrency tests use deterministic coordination rather than sleeps;
|
||||||
|
- current-state documentation describes implemented behavior rather than
|
||||||
|
referring readers to the roadmaps; and
|
||||||
|
- the feature and implementation roadmaps contain no unresolved work marked
|
||||||
|
complete.
|
||||||
|
|
||||||
|
### Completion Gate
|
||||||
|
|
||||||
|
The implementation is complete only when every target-end-state item in
|
||||||
|
`concurrency.md` is implemented, race-enabled tests demonstrate the configured
|
||||||
|
limits and cancellation safety, durable contracts no longer depend on roadmap
|
||||||
|
prose, and the OpenRouter compatibility change is clearly reported for the
|
||||||
|
next minor release.
|
||||||
|
|
||||||
|
## Open Questions
|
||||||
|
|
||||||
|
None. The feature roadmap and this plan fix the public representation,
|
||||||
|
registry defaults, admission bound, FIFO generation behavior, early-routing
|
||||||
|
refactor, cancellation races, error identities, engine and repair lifetimes,
|
||||||
|
test ownership, compatibility treatment, and non-goals required for
|
||||||
|
implementation.
|
||||||
38
engine.go
38
engine.go
@@ -12,7 +12,9 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
artifactadapter "gitea.maximumdirect.net/eric/promptkit/internal/artifact"
|
artifactadapter "gitea.maximumdirect.net/eric/promptkit/internal/artifact"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile/builtin"
|
"gitea.maximumdirect.net/eric/promptkit/internal/profile/builtin"
|
||||||
@@ -23,7 +25,8 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// ErrInvalidConfig identifies invalid engine construction, including missing
|
// ErrInvalidConfig identifies invalid engine construction, including missing
|
||||||
// required configuration, invalid options, and a nil Engine receiver.
|
// required configuration, invalid options or backend registrations, and a nil
|
||||||
|
// Engine receiver.
|
||||||
var ErrInvalidConfig = errors.New("invalid engine configuration")
|
var ErrInvalidConfig = errors.New("invalid engine configuration")
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -46,8 +49,8 @@ var (
|
|||||||
// ErrPromptNotFound.
|
// ErrPromptNotFound.
|
||||||
ErrPromptLoad = errors.New("failed to load prompt definition")
|
ErrPromptLoad = errors.New("failed to load prompt definition")
|
||||||
// ErrProfileLoad identifies a failure to read, decode, validate, or select
|
// ErrProfileLoad identifies a failure to read, decode, validate, or select
|
||||||
// an execution profile, except for the not-found case represented by
|
// an execution profile or resolve its backend, except for the profile
|
||||||
// ErrProfileNotFound.
|
// not-found case represented by ErrProfileNotFound.
|
||||||
ErrProfileLoad = errors.New("failed to load execution profile")
|
ErrProfileLoad = errors.New("failed to load execution profile")
|
||||||
// ErrAPIKeyEnvMissing identifies an APIKeyEnv whose environment variable is
|
// ErrAPIKeyEnvMissing identifies an APIKeyEnv whose environment variable is
|
||||||
// unset or empty when no direct RunRequest.APIKey takes precedence. Such an
|
// unset or empty when no direct RunRequest.APIKey takes precedence. Such an
|
||||||
@@ -107,8 +110,10 @@ type Config struct {
|
|||||||
// NewEngine applies options in argument order and ignores nil options. Within
|
// NewEngine applies options in argument order and ignores nil options. Within
|
||||||
// each prompt-source, profile-source, in-memory-profile, schema-source,
|
// each prompt-source, profile-source, in-memory-profile, schema-source,
|
||||||
// model-client, and artifact-reader category, the last non-nil valid option
|
// model-client, and artifact-reader category, the last non-nil valid option
|
||||||
// replaces earlier options in that category. An invalid option fails
|
// replaces earlier options in that category. WithBackend is the additive
|
||||||
// construction even if a later option would replace it.
|
// exception: unique registrations accumulate, and a repeated backend ID is an
|
||||||
|
// error rather than a replacement. An invalid option fails construction even
|
||||||
|
// if a later option would replace it.
|
||||||
type Option interface {
|
type Option interface {
|
||||||
apply(*engineOptions) error
|
apply(*engineOptions) error
|
||||||
}
|
}
|
||||||
@@ -125,6 +130,7 @@ type engineOptions struct {
|
|||||||
promptDefs promptdef.Repository
|
promptDefs promptdef.Repository
|
||||||
profiles profile.Repository
|
profiles profile.Repository
|
||||||
memoryProfiles profile.Repository
|
memoryProfiles profile.Repository
|
||||||
|
backends []domain.Backend
|
||||||
validator validate.Validator
|
validator validate.Validator
|
||||||
promptSource bool
|
promptSource bool
|
||||||
profileSource bool
|
profileSource bool
|
||||||
@@ -302,8 +308,9 @@ func WithSchemaFile(path string) Option {
|
|||||||
//
|
//
|
||||||
// Options are applied in order according to [Option]. PromptDir is required
|
// Options are applied in order according to [Option]. PromptDir is required
|
||||||
// unless a prompt-source option is present. Construction validates option
|
// unless a prompt-source option is present. Construction validates option
|
||||||
// arguments and in-memory profiles but defers reading and validating prompt,
|
// arguments, in-memory profiles, and backend registrations but defers reading
|
||||||
// file-backed profile, and schema contents until Prepare or Run needs them.
|
// and validating prompt, file-backed profile, and schema contents until Prepare
|
||||||
|
// or Run needs them.
|
||||||
//
|
//
|
||||||
// NewEngine returns an error matching ErrInvalidConfig for invalid
|
// NewEngine returns an error matching ErrInvalidConfig for invalid
|
||||||
// configuration or options. It does not perform model requests or require
|
// configuration or options. It does not perform model requests or require
|
||||||
@@ -335,6 +342,11 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
|||||||
profiles = profile.NewOverlayRepository(options.memoryProfiles, profiles)
|
profiles = profile.NewOverlayRepository(options.memoryProfiles, profiles)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
backendRegistry, err := backend.NewRegistry(options.backends)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: failed to construct backend registry: %v", ErrInvalidConfig, err)
|
||||||
|
}
|
||||||
|
|
||||||
validator := options.validator
|
validator := options.validator
|
||||||
if !options.validatorSource {
|
if !options.validatorSource {
|
||||||
schemaDir := cfg.SchemaDir
|
schemaDir := cfg.SchemaDir
|
||||||
@@ -365,6 +377,7 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
|||||||
runner: usecase.NewRunner(
|
runner: usecase.NewRunner(
|
||||||
promptDefs,
|
promptDefs,
|
||||||
profiles,
|
profiles,
|
||||||
|
backendRegistry,
|
||||||
artifacts,
|
artifacts,
|
||||||
prompt.NewGoRenderer(),
|
prompt.NewGoRenderer(),
|
||||||
llmClient,
|
llmClient,
|
||||||
@@ -395,11 +408,12 @@ func fileSource(name string) (fs.FS, string, error) {
|
|||||||
|
|
||||||
// Prepare resolves and renders a prompt request without calling an LLM.
|
// Prepare resolves and renders a prompt request without calling an LLM.
|
||||||
//
|
//
|
||||||
// Prepare selects the prompt and profile, resolves effective execution
|
// Prepare selects the prompt and profile, resolves any selected backend and
|
||||||
// settings and the output contract, loads and hashes inputs, loads structured
|
// effective execution settings, resolves the output contract, loads and hashes
|
||||||
// output schema metadata when required, and renders the session ID and
|
// inputs, loads structured-output schema metadata when required, and renders
|
||||||
// messages. The returned PreparedRun is owned by the caller and never contains
|
// the session ID and messages. The returned PreparedRun is owned by the caller
|
||||||
// a resolved API-key value, model output, or validation result.
|
// and never contains a resolved API-key value, model output, or validation
|
||||||
|
// result.
|
||||||
//
|
//
|
||||||
// A nil Engine returns an error matching ErrInvalidConfig. Request and
|
// A nil Engine returns an error matching ErrInvalidConfig. Request and
|
||||||
// preparation failures may match ErrInvalidRequest, ErrPromptNotFound,
|
// preparation failures may match ErrInvalidRequest, ErrPromptNotFound,
|
||||||
|
|||||||
211
engine_test.go
211
engine_test.go
@@ -281,6 +281,9 @@ func TestEngineExecutionSettingPrecedence(t *testing.T) {
|
|||||||
intPointer := func(value int) *int {
|
intPointer := func(value int) *int {
|
||||||
return &value
|
return &value
|
||||||
}
|
}
|
||||||
|
stringPointer := func(value string) *string {
|
||||||
|
return &value
|
||||||
|
}
|
||||||
|
|
||||||
defaultsProfile := executionProfileFixture{
|
defaultsProfile := executionProfileFixture{
|
||||||
id: "settings-defaults",
|
id: "settings-defaults",
|
||||||
@@ -388,7 +391,7 @@ func TestEngineExecutionSettingPrecedence(t *testing.T) {
|
|||||||
TopP: floatPointer(requestTarget.TopP),
|
TopP: floatPointer(requestTarget.TopP),
|
||||||
TimeoutSeconds: intPointer(requestTarget.TimeoutSeconds),
|
TimeoutSeconds: intPointer(requestTarget.TimeoutSeconds),
|
||||||
ServiceTier: requestTarget.ServiceTier,
|
ServiceTier: requestTarget.ServiceTier,
|
||||||
ReasoningEffort: requestTarget.ReasoningEffort,
|
ReasoningEffort: stringPointer(requestTarget.ReasoningEffort),
|
||||||
APIKeyEnv: requestTarget.APIKeyEnv,
|
APIKeyEnv: requestTarget.APIKeyEnv,
|
||||||
ExtraParams: requestTarget.ExtraParams,
|
ExtraParams: requestTarget.ExtraParams,
|
||||||
},
|
},
|
||||||
@@ -452,6 +455,43 @@ func TestEngineExecutionSettingPrecedence(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
t.Run("blank request reasoning clears profile setting", func(t *testing.T) {
|
||||||
|
profile := executionProfileFixture{
|
||||||
|
id: "settings-reasoning-clear",
|
||||||
|
endpoint: "http://profile-reasoning.test/v1",
|
||||||
|
model: "profile-reasoning-model",
|
||||||
|
reasoningEffort: "medium",
|
||||||
|
}
|
||||||
|
profileDir := t.TempDir()
|
||||||
|
writeExecutionProfileFixture(t, profileDir, profile)
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
|
PromptDir: frameworkPromptDir,
|
||||||
|
ProfileDir: profileDir,
|
||||||
|
SchemaDir: frameworkSchemaDir,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||||
|
PromptID: frameworkMarkdownSummaryPromptID,
|
||||||
|
ProfileID: profile.id,
|
||||||
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
|
"transcript": promptkit.Inline("Nia labels the archive."),
|
||||||
|
"glossary": promptkit.Inline("archive: A catalogued collection."),
|
||||||
|
},
|
||||||
|
Execution: &promptkit.ExecutionTargetOverride{
|
||||||
|
ReasoningEffort: stringPointer(" \t "),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare engine: %v", err)
|
||||||
|
}
|
||||||
|
if prepared.EffectiveModelParams.ReasoningEffort != "" {
|
||||||
|
t.Fatalf("expected blank request reasoning to clear profile value, got %q", prepared.EffectiveModelParams.ReasoningEffort)
|
||||||
|
}
|
||||||
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunSucceedsWithInjectedLLMClient(t *testing.T) {
|
func TestRunSucceedsWithInjectedLLMClient(t *testing.T) {
|
||||||
@@ -571,19 +611,26 @@ func TestEngineRunWithDirectorySourcesAndFileInputs(t *testing.T) {
|
|||||||
|
|
||||||
func TestRunPassesPreparedRequestToInjectedLLMClient(t *testing.T) {
|
func TestRunPassesPreparedRequestToInjectedLLMClient(t *testing.T) {
|
||||||
const directKey = "direct-injected-key"
|
const directKey = "direct-injected-key"
|
||||||
|
const directSession = "assembled-session"
|
||||||
fake := &fakeLLMClient{
|
fake := &fakeLLMClient{
|
||||||
response: &promptkit.GenerateResponse{Content: `{"events":[{"title":"Archive labelled"}]}`},
|
response: &promptkit.GenerateResponse{Content: `{"events":[{"title":"Archive labelled"}]}`},
|
||||||
}
|
}
|
||||||
engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(fake))
|
engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(fake))
|
||||||
|
|
||||||
_, err := engine.Run(context.Background(), promptkit.RunRequest{
|
runRequest := promptkit.RunRequest{
|
||||||
PromptID: frameworkStructuredEventsPromptID,
|
PromptID: frameworkStructuredEventsPromptID,
|
||||||
|
SessionID: " " + directSession + " ",
|
||||||
APIKey: directKey,
|
APIKey: directKey,
|
||||||
Inputs: map[string]promptkit.ArtifactRef{
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
"transcript": promptkit.Inline("Rin opens the gate."),
|
"transcript": promptkit.Inline("Rin opens the gate."),
|
||||||
"glossary": promptkit.Inline("gate: A guarded passage."),
|
"glossary": promptkit.Inline("gate: A guarded passage."),
|
||||||
},
|
},
|
||||||
})
|
}
|
||||||
|
prepared, err := engine.Prepare(context.Background(), runRequest)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected prepare to succeed, got %v", err)
|
||||||
|
}
|
||||||
|
result, err := engine.Run(context.Background(), runRequest)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected run to succeed, got %v", err)
|
t.Fatalf("expected run to succeed, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -594,6 +641,16 @@ func TestRunPassesPreparedRequestToInjectedLLMClient(t *testing.T) {
|
|||||||
if len(req.Prompt.Messages) != 2 || !strings.Contains(req.Prompt.Messages[1].Content, "Rin opens the gate.") {
|
if len(req.Prompt.Messages) != 2 || !strings.Contains(req.Prompt.Messages[1].Content, "Rin opens the gate.") {
|
||||||
t.Fatalf("expected rendered prompt in generate request, got %+v", req.Prompt)
|
t.Fatalf("expected rendered prompt in generate request, got %+v", req.Prompt)
|
||||||
}
|
}
|
||||||
|
if prepared.SessionID != directSession ||
|
||||||
|
req.Prompt.SessionID != directSession ||
|
||||||
|
result.SessionID != directSession {
|
||||||
|
t.Fatalf(
|
||||||
|
"direct session did not propagate consistently: prepared=%q generated=%q result=%q",
|
||||||
|
prepared.SessionID,
|
||||||
|
req.Prompt.SessionID,
|
||||||
|
result.SessionID,
|
||||||
|
)
|
||||||
|
}
|
||||||
if req.StructuredOutput == nil || req.StructuredOutput.Type != promptkit.StructuredOutputJSONSchema || req.StructuredOutput.JSONSchema == nil {
|
if req.StructuredOutput == nil || req.StructuredOutput.Type != promptkit.StructuredOutputJSONSchema || req.StructuredOutput.JSONSchema == nil {
|
||||||
t.Fatalf("expected structured output handoff, got %+v", req.StructuredOutput)
|
t.Fatalf("expected structured output handoff, got %+v", req.StructuredOutput)
|
||||||
}
|
}
|
||||||
@@ -755,6 +812,92 @@ func TestRunUsesDirectAPIKeyWithDefaultLLMClient(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunUsesResolvedBackendWithBuiltInLLMClient(t *testing.T) {
|
||||||
|
const (
|
||||||
|
backendID = "local-test"
|
||||||
|
envName = "PROMPTKIT_BACKEND_TRANSPORT_KEY"
|
||||||
|
apiKey = "synthetic-backend-key"
|
||||||
|
)
|
||||||
|
t.Setenv(envName, apiKey)
|
||||||
|
|
||||||
|
var (
|
||||||
|
gotAuth string
|
||||||
|
gotBody map[string]any
|
||||||
|
)
|
||||||
|
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
gotAuth = r.Header.Get("Authorization")
|
||||||
|
if r.URL.Path != "/v1/chat/completions" {
|
||||||
|
t.Errorf("unexpected path: %s", r.URL.Path)
|
||||||
|
}
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&gotBody); err != nil {
|
||||||
|
t.Errorf("decode request body: %v", err)
|
||||||
|
}
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_, _ = w.Write([]byte(`{
|
||||||
|
"choices": [{"message": {"role": "assistant", "content": "# Summary\n\nDone."}}],
|
||||||
|
"usage": {"prompt_tokens": 3, "completion_tokens": 4, "total_tokens": 7}
|
||||||
|
}`))
|
||||||
|
}))
|
||||||
|
defer server.Close()
|
||||||
|
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||||
|
PromptDir: frameworkPromptDir,
|
||||||
|
SchemaDir: frameworkSchemaDir,
|
||||||
|
},
|
||||||
|
promptkit.WithBackend(promptkit.Backend{
|
||||||
|
ID: backendID,
|
||||||
|
Endpoint: server.URL + "/v1",
|
||||||
|
APIKeyEnv: envName,
|
||||||
|
ExtraParams: map[string]any{
|
||||||
|
"provider": "synthetic",
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "backend-transport",
|
||||||
|
BackendID: backendID,
|
||||||
|
Model: "test-model",
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := engine.Run(context.Background(), promptkit.RunRequest{
|
||||||
|
PromptID: frameworkMarkdownSummaryPromptID,
|
||||||
|
ProfileID: "backend-transport",
|
||||||
|
Inputs: map[string]promptkit.ArtifactRef{
|
||||||
|
"transcript": promptkit.Inline("Rin opens the gate."),
|
||||||
|
"glossary": promptkit.Inline("gate: A guarded passage."),
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("run with resolved backend: %v", err)
|
||||||
|
}
|
||||||
|
if gotAuth != "Bearer "+apiKey {
|
||||||
|
t.Fatalf("unexpected Authorization header: %q", gotAuth)
|
||||||
|
}
|
||||||
|
if gotBody["model"] != "test-model" || gotBody["provider"] != "synthetic" {
|
||||||
|
t.Fatalf("backend defaults did not reach provider payload: %#v", gotBody)
|
||||||
|
}
|
||||||
|
for _, field := range []string{"backend_id", "api_key_env"} {
|
||||||
|
if _, ok := gotBody[field]; ok {
|
||||||
|
t.Fatalf("internal metadata field %q was serialized to provider payload: %#v", field, gotBody)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
bodyJSON, err := json.Marshal(gotBody)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal captured provider payload: %v", err)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(bodyJSON), apiKey) {
|
||||||
|
t.Fatalf("credential value was serialized to provider payload: %s", bodyJSON)
|
||||||
|
}
|
||||||
|
if result.SelectedBackendID != backendID ||
|
||||||
|
result.EffectiveModelParams.Endpoint != server.URL+"/v1" ||
|
||||||
|
result.EffectiveModelParams.APIKeyEnv != envName {
|
||||||
|
t.Fatalf("unexpected resolved backend metadata: %+v", result)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestPrepareDirectAPIKeyBypassesMissingEnvWithoutLeakingOrHashing(t *testing.T) {
|
func TestPrepareDirectAPIKeyBypassesMissingEnvWithoutLeakingOrHashing(t *testing.T) {
|
||||||
const missingEnv = "PROMPTKIT_PUBLIC_PREPARE_MISSING"
|
const missingEnv = "PROMPTKIT_PUBLIC_PREPARE_MISSING"
|
||||||
const firstKey = "first-direct-key"
|
const firstKey = "first-direct-key"
|
||||||
@@ -1271,6 +1414,18 @@ func TestPrepareUsesBuiltInProfileWithoutProfileDir(t *testing.T) {
|
|||||||
if prepared.SelectedProfileID != "mistral-small-3" {
|
if prepared.SelectedProfileID != "mistral-small-3" {
|
||||||
t.Fatalf("unexpected selected profile: %q", prepared.SelectedProfileID)
|
t.Fatalf("unexpected selected profile: %q", prepared.SelectedProfileID)
|
||||||
}
|
}
|
||||||
|
if prepared.SelectedBackendID != promptkit.BackendOpenRouter {
|
||||||
|
t.Fatalf("unexpected selected backend: %q", prepared.SelectedBackendID)
|
||||||
|
}
|
||||||
|
if prepared.EffectiveModelParams.BackendID != promptkit.BackendOpenRouter {
|
||||||
|
t.Fatalf("unexpected effective backend: %q", prepared.EffectiveModelParams.BackendID)
|
||||||
|
}
|
||||||
|
if prepared.EffectiveModelParams.Endpoint != "https://openrouter.ai/api/v1" {
|
||||||
|
t.Fatalf("unexpected built-in endpoint: %q", prepared.EffectiveModelParams.Endpoint)
|
||||||
|
}
|
||||||
|
if prepared.EffectiveModelParams.APIKeyEnv != "OPENROUTER_API_KEY" {
|
||||||
|
t.Fatalf("unexpected built-in api key environment name: %q", prepared.EffectiveModelParams.APIKeyEnv)
|
||||||
|
}
|
||||||
if prepared.EffectiveModelParams.Model != "mistralai/mistral-small-3.2-24b-instruct" {
|
if prepared.EffectiveModelParams.Model != "mistralai/mistral-small-3.2-24b-instruct" {
|
||||||
t.Fatalf("unexpected built-in model: %q", prepared.EffectiveModelParams.Model)
|
t.Fatalf("unexpected built-in model: %q", prepared.EffectiveModelParams.Model)
|
||||||
}
|
}
|
||||||
@@ -1641,6 +1796,7 @@ func TestOpenAICompatibleProfileRunsThroughNormalProfilePath(t *testing.T) {
|
|||||||
fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
|
fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
|
||||||
prof := promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
prof := promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||||
ID: "template-profile",
|
ID: "template-profile",
|
||||||
|
BackendID: " openrouter ",
|
||||||
Endpoint: "http://template/v1",
|
Endpoint: "http://template/v1",
|
||||||
Model: "template-model",
|
Model: "template-model",
|
||||||
APIKeyRequired: true,
|
APIKeyRequired: true,
|
||||||
@@ -1672,7 +1828,9 @@ func TestOpenAICompatibleProfileRunsThroughNormalProfilePath(t *testing.T) {
|
|||||||
if len(fake.requests) != 1 {
|
if len(fake.requests) != 1 {
|
||||||
t.Fatalf("expected one request, got %d", len(fake.requests))
|
t.Fatalf("expected one request, got %d", len(fake.requests))
|
||||||
}
|
}
|
||||||
if fake.requests[0].Target.Model != "template-model" || fake.requests[0].APIKey != "template-key" {
|
if fake.requests[0].Target.BackendID != promptkit.BackendOpenRouter ||
|
||||||
|
fake.requests[0].Target.Model != "template-model" ||
|
||||||
|
fake.requests[0].APIKey != "template-key" {
|
||||||
t.Fatalf("unexpected generated request: %+v", fake.requests[0])
|
t.Fatalf("unexpected generated request: %+v", fake.requests[0])
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(fake.requests[0].Target.ExtraParams, map[string]any{"provider": "template"}) {
|
if !reflect.DeepEqual(fake.requests[0].Target.ExtraParams, map[string]any{"provider": "template"}) {
|
||||||
@@ -2256,6 +2414,11 @@ func TestExtraParamsTypedNestedValuesAreCopiedAcrossPublicBoundary(t *testing.T)
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunRejectsInvalidExtraParams(t *testing.T) {
|
func TestRunRejectsInvalidExtraParams(t *testing.T) {
|
||||||
|
cyclicMap := map[string]any{}
|
||||||
|
cyclicMap["self"] = cyclicMap
|
||||||
|
cyclicSlice := []any{nil}
|
||||||
|
cyclicSlice[0] = cyclicSlice
|
||||||
|
|
||||||
tests := []struct {
|
tests := []struct {
|
||||||
name string
|
name string
|
||||||
extraParams map[string]any
|
extraParams map[string]any
|
||||||
@@ -2267,42 +2430,10 @@ func TestRunRejectsInvalidExtraParams(t *testing.T) {
|
|||||||
{name: "nan", extraParams: map[string]any{"bad": math.NaN()}},
|
{name: "nan", extraParams: map[string]any{"bad": math.NaN()}},
|
||||||
{name: "positive infinity", extraParams: map[string]any{"bad": math.Inf(1)}},
|
{name: "positive infinity", extraParams: map[string]any{"bad": math.Inf(1)}},
|
||||||
{name: "negative infinity", extraParams: map[string]any{"bad": math.Inf(-1)}},
|
{name: "negative infinity", extraParams: map[string]any{"bad": math.Inf(-1)}},
|
||||||
}
|
{name: "cyclic map", extraParams: cyclicMap},
|
||||||
for _, tc := range tests {
|
{name: "cyclic slice", extraParams: map[string]any{"cycle": cyclicSlice}},
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
{name: "malformed JSON number", extraParams: map[string]any{"value": json.Number("+1")}},
|
||||||
fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
|
{name: "empty nested key", extraParams: map[string]any{"nested": map[string]any{"": true}}},
|
||||||
engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(fake))
|
|
||||||
|
|
||||||
_, err := engine.Run(context.Background(), promptkit.RunRequest{
|
|
||||||
PromptID: frameworkMarkdownSummaryPromptID,
|
|
||||||
Inputs: map[string]promptkit.ArtifactRef{
|
|
||||||
"transcript": promptkit.Inline("Rin opens the gate."),
|
|
||||||
"glossary": promptkit.Inline("gate: A guarded passage."),
|
|
||||||
},
|
|
||||||
Execution: &promptkit.ExecutionTargetOverride{ExtraParams: tc.extraParams},
|
|
||||||
})
|
|
||||||
if !errors.Is(err, promptkit.ErrInvalidRequest) {
|
|
||||||
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
|
||||||
}
|
|
||||||
if len(fake.requests) != 0 {
|
|
||||||
t.Fatalf("expected invalid request to fail before LLM call, got %d requests", len(fake.requests))
|
|
||||||
}
|
|
||||||
})
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func TestRunRejectsCyclicExtraParams(t *testing.T) {
|
|
||||||
cyclicMap := map[string]any{}
|
|
||||||
cyclicMap["self"] = cyclicMap
|
|
||||||
cyclicSlice := []any{nil}
|
|
||||||
cyclicSlice[0] = cyclicSlice
|
|
||||||
|
|
||||||
tests := []struct {
|
|
||||||
name string
|
|
||||||
extraParams map[string]any
|
|
||||||
}{
|
|
||||||
{name: "map", extraParams: cyclicMap},
|
|
||||||
{name: "slice", extraParams: map[string]any{"cycle": cyclicSlice}},
|
|
||||||
}
|
}
|
||||||
for _, tc := range tests {
|
for _, tc := range tests {
|
||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
|||||||
155
internal/backend/registry.go
Normal file
155
internal/backend/registry.go
Normal file
@@ -0,0 +1,155 @@
|
|||||||
|
// Package backend owns validated, immutable OpenAI-compatible backend
|
||||||
|
// definitions.
|
||||||
|
package backend
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"net/url"
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
// OpenRouterID is the reserved ID of Promptkit's built-in OpenRouter
|
||||||
|
// backend.
|
||||||
|
OpenRouterID = "openrouter"
|
||||||
|
|
||||||
|
openRouterEndpoint = "https://openrouter.ai/api/v1"
|
||||||
|
openRouterAPIKeyEnv = "OPENROUTER_API_KEY"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ErrBackendNotFound identifies a registry lookup for an unknown backend ID.
|
||||||
|
var ErrBackendNotFound = errors.New("backend not found")
|
||||||
|
|
||||||
|
var environmentVariableName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||||
|
|
||||||
|
// Registry is an immutable collection of validated backend definitions.
|
||||||
|
type Registry struct {
|
||||||
|
backends map[string]domain.Backend
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewRegistry constructs a registry containing the built-in OpenRouter
|
||||||
|
// definition followed by the supplied additions. Every ID must be unique.
|
||||||
|
func NewRegistry(additions []domain.Backend) (*Registry, error) {
|
||||||
|
registry := &Registry{
|
||||||
|
backends: make(map[string]domain.Backend, len(additions)+1),
|
||||||
|
}
|
||||||
|
|
||||||
|
definitions := make([]domain.Backend, 0, len(additions)+1)
|
||||||
|
definitions = append(definitions, domain.Backend{
|
||||||
|
ID: OpenRouterID,
|
||||||
|
Endpoint: openRouterEndpoint,
|
||||||
|
APIKeyEnv: openRouterAPIKeyEnv,
|
||||||
|
})
|
||||||
|
definitions = append(definitions, additions...)
|
||||||
|
|
||||||
|
for _, definition := range definitions {
|
||||||
|
definition.ID = strings.TrimSpace(definition.ID)
|
||||||
|
if definition.ID == "" {
|
||||||
|
return nil, errors.New("backend ID must not be blank")
|
||||||
|
}
|
||||||
|
if _, exists := registry.backends[definition.ID]; exists {
|
||||||
|
return nil, fmt.Errorf("backend ID %q is already registered", definition.ID)
|
||||||
|
}
|
||||||
|
|
||||||
|
normalized, err := normalizeBackend(definition)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
registry.backends[normalized.ID] = normalized
|
||||||
|
}
|
||||||
|
|
||||||
|
return registry, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetBackend returns a defensive copy of the backend registered with id.
|
||||||
|
func (r *Registry) GetBackend(id string) (domain.Backend, error) {
|
||||||
|
if r == nil {
|
||||||
|
return domain.Backend{}, fmt.Errorf("%w: %q", ErrBackendNotFound, id)
|
||||||
|
}
|
||||||
|
definition, ok := r.backends[id]
|
||||||
|
if !ok {
|
||||||
|
return domain.Backend{}, fmt.Errorf("%w: %q", ErrBackendNotFound, id)
|
||||||
|
}
|
||||||
|
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
|
||||||
|
if err != nil {
|
||||||
|
return domain.Backend{}, fmt.Errorf("copy backend %q: %w", id, err)
|
||||||
|
}
|
||||||
|
definition.ExtraParams = extraParams
|
||||||
|
return definition, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func normalizeBackend(definition domain.Backend) (domain.Backend, error) {
|
||||||
|
definition.Endpoint = strings.TrimSpace(definition.Endpoint)
|
||||||
|
if err := validateEndpoint(definition.Endpoint); err != nil {
|
||||||
|
return domain.Backend{}, fmt.Errorf("backend %q endpoint: %w", definition.ID, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
definition.APIKeyEnv = strings.TrimSpace(definition.APIKeyEnv)
|
||||||
|
if definition.APIKeyEnv != "" && !environmentVariableName.MatchString(definition.APIKeyEnv) {
|
||||||
|
return domain.Backend{}, fmt.Errorf(
|
||||||
|
"backend %q api key environment variable %q is invalid",
|
||||||
|
definition.ID,
|
||||||
|
definition.APIKeyEnv,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
|
keys := make([]string, 0, len(definition.ExtraParams))
|
||||||
|
for key := range definition.ExtraParams {
|
||||||
|
keys = append(keys, key)
|
||||||
|
}
|
||||||
|
sort.Strings(keys)
|
||||||
|
for _, key := range keys {
|
||||||
|
if key == "" {
|
||||||
|
return domain.Backend{}, fmt.Errorf("backend %q extra parameter key must not be empty", definition.ID)
|
||||||
|
}
|
||||||
|
if llm.IsReservedOpenAIChatRequestField(key) {
|
||||||
|
return domain.Backend{}, fmt.Errorf(
|
||||||
|
"backend %q extra parameter %q collides with a reserved request field",
|
||||||
|
definition.ID,
|
||||||
|
key,
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
|
||||||
|
if err != nil {
|
||||||
|
return domain.Backend{}, fmt.Errorf("backend %q extra parameters: %w", definition.ID, err)
|
||||||
|
}
|
||||||
|
definition.ExtraParams = extraParams
|
||||||
|
return definition, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func validateEndpoint(endpoint string) error {
|
||||||
|
if endpoint == "" {
|
||||||
|
return errors.New("must not be blank")
|
||||||
|
}
|
||||||
|
if strings.Contains(endpoint, "#") {
|
||||||
|
return errors.New("must not contain a fragment")
|
||||||
|
}
|
||||||
|
|
||||||
|
parsed, err := url.Parse(endpoint)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("must be a valid URL: %w", err)
|
||||||
|
}
|
||||||
|
scheme := strings.ToLower(parsed.Scheme)
|
||||||
|
if scheme != "http" && scheme != "https" {
|
||||||
|
return errors.New("must use http or https")
|
||||||
|
}
|
||||||
|
if !parsed.IsAbs() || parsed.Hostname() == "" {
|
||||||
|
return errors.New("must be absolute and include a host")
|
||||||
|
}
|
||||||
|
if parsed.User != nil {
|
||||||
|
return errors.New("must not contain user information")
|
||||||
|
}
|
||||||
|
if parsed.RawQuery != "" || parsed.ForceQuery {
|
||||||
|
return errors.New("must not contain a query string")
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
225
internal/backend/registry_test.go
Normal file
225
internal/backend/registry_test.go
Normal file
@@ -0,0 +1,225 @@
|
|||||||
|
package backend_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"errors"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
)
|
||||||
|
|
||||||
|
const validEndpoint = "https://backend.example/v1"
|
||||||
|
|
||||||
|
func TestRegistryIncludesExactOpenRouterDefinition(t *testing.T) {
|
||||||
|
registry, err := backend.NewRegistry(nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct registry: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
definition, err := registry.GetBackend(backend.OpenRouterID)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("look up OpenRouter: %v", err)
|
||||||
|
}
|
||||||
|
if definition.ID != "openrouter" ||
|
||||||
|
definition.Endpoint != "https://openrouter.ai/api/v1" ||
|
||||||
|
definition.APIKeyEnv != "OPENROUTER_API_KEY" ||
|
||||||
|
definition.ExtraParams != nil {
|
||||||
|
t.Fatalf("unexpected OpenRouter definition: %#v", definition)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRegistryNormalizesUniqueAdditionsAndIsolatesMutations(t *testing.T) {
|
||||||
|
nested := map[string]int{"limit": 2}
|
||||||
|
extraParams := map[string]any{
|
||||||
|
"count": int64(7),
|
||||||
|
"nested": nested,
|
||||||
|
}
|
||||||
|
registry, err := backend.NewRegistry([]domain.Backend{
|
||||||
|
{
|
||||||
|
ID: " custom ",
|
||||||
|
Endpoint: " https://custom.example/openai/v1 ",
|
||||||
|
APIKeyEnv: " CUSTOM_API_KEY ",
|
||||||
|
ExtraParams: extraParams,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ID: "Custom",
|
||||||
|
Endpoint: validEndpoint,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct registry: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
nested["limit"] = 99
|
||||||
|
extraParams["added"] = true
|
||||||
|
|
||||||
|
got, err := registry.GetBackend("custom")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("look up custom backend: %v", err)
|
||||||
|
}
|
||||||
|
if got.ID != "custom" ||
|
||||||
|
got.Endpoint != "https://custom.example/openai/v1" ||
|
||||||
|
got.APIKeyEnv != "CUSTOM_API_KEY" {
|
||||||
|
t.Fatalf("unexpected normalized definition: %#v", got)
|
||||||
|
}
|
||||||
|
if count, ok := got.ExtraParams["count"].(int64); !ok || count != 7 {
|
||||||
|
t.Fatalf("integer type or value changed: %#v", got.ExtraParams["count"])
|
||||||
|
}
|
||||||
|
gotNested, ok := got.ExtraParams["nested"].(map[string]int)
|
||||||
|
if !ok || gotNested["limit"] != 2 {
|
||||||
|
t.Fatalf("container type or value changed: %#v", got.ExtraParams["nested"])
|
||||||
|
}
|
||||||
|
if _, exists := got.ExtraParams["added"]; exists {
|
||||||
|
t.Fatalf("registry retained caller map: %#v", got.ExtraParams)
|
||||||
|
}
|
||||||
|
|
||||||
|
gotNested["limit"] = 100
|
||||||
|
got.ExtraParams["added"] = true
|
||||||
|
again, err := registry.GetBackend("custom")
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("look up custom backend again: %v", err)
|
||||||
|
}
|
||||||
|
if again.ExtraParams["nested"].(map[string]int)["limit"] != 2 {
|
||||||
|
t.Fatalf("lookup exposed registry nested map: %#v", again.ExtraParams)
|
||||||
|
}
|
||||||
|
if _, exists := again.ExtraParams["added"]; exists {
|
||||||
|
t.Fatalf("lookup exposed registry map: %#v", again.ExtraParams)
|
||||||
|
}
|
||||||
|
|
||||||
|
if _, err := registry.GetBackend("Custom"); err != nil {
|
||||||
|
t.Fatalf("backend IDs should be case-sensitive: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewRegistryRejectsDuplicateIDs(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
additions []domain.Backend
|
||||||
|
wantID string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "built-in collision after normalization",
|
||||||
|
additions: []domain.Backend{{
|
||||||
|
ID: " openrouter ",
|
||||||
|
}},
|
||||||
|
wantID: "openrouter",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "consumer collision after normalization",
|
||||||
|
additions: []domain.Backend{
|
||||||
|
{ID: "custom", Endpoint: validEndpoint},
|
||||||
|
{ID: " custom ", Endpoint: "https://other.example/v1"},
|
||||||
|
},
|
||||||
|
wantID: "custom",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
_, err := backend.NewRegistry(tc.additions)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected duplicate ID error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), tc.wantID) {
|
||||||
|
t.Fatalf("expected error to identify %q, got %v", tc.wantID, err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewRegistryValidatesIDs(t *testing.T) {
|
||||||
|
for _, id := range []string{"", " \t\n "} {
|
||||||
|
t.Run(id, func(t *testing.T) {
|
||||||
|
_, err := backend.NewRegistry([]domain.Backend{{
|
||||||
|
ID: id,
|
||||||
|
Endpoint: validEndpoint,
|
||||||
|
}})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected blank ID error")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewRegistryValidatesEndpoints(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
endpoint string
|
||||||
|
}{
|
||||||
|
{name: "blank", endpoint: ""},
|
||||||
|
{name: "relative", endpoint: "/v1"},
|
||||||
|
{name: "missing host", endpoint: "https:///v1"},
|
||||||
|
{name: "unsupported scheme", endpoint: "ftp://backend.example/v1"},
|
||||||
|
{name: "user information", endpoint: "https://user@backend.example/v1"},
|
||||||
|
{name: "query", endpoint: "https://backend.example/v1?mode=chat"},
|
||||||
|
{name: "empty query", endpoint: "https://backend.example/v1?"},
|
||||||
|
{name: "fragment", endpoint: "https://backend.example/v1#chat"},
|
||||||
|
{name: "empty fragment", endpoint: "https://backend.example/v1#"},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
_, err := backend.NewRegistry([]domain.Backend{{
|
||||||
|
ID: "custom",
|
||||||
|
Endpoint: tc.endpoint,
|
||||||
|
}})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected invalid endpoint error")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewRegistryValidatesEnvironmentVariableNames(t *testing.T) {
|
||||||
|
for _, name := range []string{"1API_KEY", "API-KEY", "API KEY", "ÅPI_KEY"} {
|
||||||
|
t.Run(name, func(t *testing.T) {
|
||||||
|
_, err := backend.NewRegistry([]domain.Backend{{
|
||||||
|
ID: "custom",
|
||||||
|
Endpoint: validEndpoint,
|
||||||
|
APIKeyEnv: name,
|
||||||
|
}})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected invalid environment-variable name error")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewRegistryRejectsInvalidAndReservedExtraParameters(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
extraParams map[string]any
|
||||||
|
}{
|
||||||
|
{name: "unsupported value", extraParams: map[string]any{"value": make(chan int)}},
|
||||||
|
{name: "reserved key", extraParams: map[string]any{"model": "override"}},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
_, err := backend.NewRegistry([]domain.Backend{{
|
||||||
|
ID: "custom",
|
||||||
|
Endpoint: validEndpoint,
|
||||||
|
ExtraParams: tc.extraParams,
|
||||||
|
}})
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected invalid extra parameters error")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRegistryLookupReportsNotFound(t *testing.T) {
|
||||||
|
registry, err := backend.NewRegistry(nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct registry: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = registry.GetBackend("missing")
|
||||||
|
if !errors.Is(err, backend.ErrBackendNotFound) {
|
||||||
|
t.Fatalf("expected ErrBackendNotFound, got %v", err)
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "missing") {
|
||||||
|
t.Fatalf("expected error to identify backend, got %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -63,6 +63,7 @@ type RunRequest struct {
|
|||||||
PromptID string
|
PromptID string
|
||||||
PromptVersion string
|
PromptVersion string
|
||||||
ProfileID string
|
ProfileID string
|
||||||
|
SessionID string
|
||||||
APIKey string `json:"-" yaml:"-"`
|
APIKey string `json:"-" yaml:"-"`
|
||||||
Inputs map[string]ArtifactRef
|
Inputs map[string]ArtifactRef
|
||||||
Vars map[string]string
|
Vars map[string]string
|
||||||
@@ -79,8 +80,10 @@ type RunResult struct {
|
|||||||
PromptID string
|
PromptID string
|
||||||
PromptVersion string
|
PromptVersion string
|
||||||
PromptHash string
|
PromptHash string
|
||||||
|
SessionID string
|
||||||
RenderedPromptHash string
|
RenderedPromptHash string
|
||||||
SelectedProfileID string
|
SelectedProfileID string
|
||||||
|
SelectedBackendID string
|
||||||
ModelName string
|
ModelName string
|
||||||
Endpoint string
|
Endpoint string
|
||||||
EffectiveModelParams ExecutionTarget
|
EffectiveModelParams ExecutionTarget
|
||||||
@@ -98,6 +101,7 @@ type PreparedRun struct {
|
|||||||
PromptVersion string `json:"prompt_version,omitempty"`
|
PromptVersion string `json:"prompt_version,omitempty"`
|
||||||
PromptHash string `json:"prompt_hash,omitempty"`
|
PromptHash string `json:"prompt_hash,omitempty"`
|
||||||
SelectedProfileID string `json:"selected_profile_id"`
|
SelectedProfileID string `json:"selected_profile_id"`
|
||||||
|
SelectedBackendID string `json:"selected_backend_id,omitempty"`
|
||||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||||
TargetPresence ExecutionTargetPresence `json:"-"`
|
TargetPresence ExecutionTargetPresence `json:"-"`
|
||||||
OutputContract OutputContract `json:"output_contract"`
|
OutputContract OutputContract `json:"output_contract"`
|
||||||
@@ -157,9 +161,18 @@ type PromptMessageTemplate struct {
|
|||||||
CacheControl *CacheControl `yaml:"cache_control,omitempty" json:"cache_control,omitempty"`
|
CacheControl *CacheControl `yaml:"cache_control,omitempty" json:"cache_control,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Backend describes reusable OpenAI-compatible connection defaults.
|
||||||
|
type Backend struct {
|
||||||
|
ID string
|
||||||
|
Endpoint string
|
||||||
|
APIKeyEnv string
|
||||||
|
ExtraParams map[string]any
|
||||||
|
}
|
||||||
|
|
||||||
// ExecutionProfile describes how and where to execute a model.
|
// ExecutionProfile describes how and where to execute a model.
|
||||||
type ExecutionProfile struct {
|
type ExecutionProfile struct {
|
||||||
ID string `yaml:"id"`
|
ID string `yaml:"id"`
|
||||||
|
BackendID string `yaml:"backend"`
|
||||||
Endpoint string `yaml:"endpoint"`
|
Endpoint string `yaml:"endpoint"`
|
||||||
Model string `yaml:"model"`
|
Model string `yaml:"model"`
|
||||||
Temperature float64 `yaml:"temperature"`
|
Temperature float64 `yaml:"temperature"`
|
||||||
@@ -182,7 +195,7 @@ type ExecutionTargetOverride struct {
|
|||||||
TopP *float64 `json:"top_p,omitempty"`
|
TopP *float64 `json:"top_p,omitempty"`
|
||||||
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
|
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
|
||||||
ServiceTier string `json:"service_tier,omitempty"`
|
ServiceTier string `json:"service_tier,omitempty"`
|
||||||
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
ReasoningEffort *string `json:"reasoning_effort,omitempty"`
|
||||||
APIKeyEnv string `json:"api_key_env,omitempty"`
|
APIKeyEnv string `json:"api_key_env,omitempty"`
|
||||||
ExtraParams map[string]any `json:"extra_params,omitempty"`
|
ExtraParams map[string]any `json:"extra_params,omitempty"`
|
||||||
}
|
}
|
||||||
@@ -198,6 +211,7 @@ type ExecutionTargetPresence struct {
|
|||||||
|
|
||||||
// ExecutionTarget represents effective model runtime settings for a run.
|
// ExecutionTarget represents effective model runtime settings for a run.
|
||||||
type ExecutionTarget struct {
|
type ExecutionTarget struct {
|
||||||
|
BackendID string `yaml:"backend" json:"backend_id,omitempty"`
|
||||||
Endpoint string `yaml:"endpoint" json:"endpoint"`
|
Endpoint string `yaml:"endpoint" json:"endpoint"`
|
||||||
Model string `yaml:"model" json:"model"`
|
Model string `yaml:"model" json:"model"`
|
||||||
Temperature float64 `yaml:"temperature" json:"temperature"`
|
Temperature float64 `yaml:"temperature" json:"temperature"`
|
||||||
|
|||||||
19
internal/domain/session.go
Normal file
19
internal/domain/session.go
Normal file
@@ -0,0 +1,19 @@
|
|||||||
|
package domain
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
"unicode/utf8"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NormalizeSessionID applies the shared session identifier rule.
|
||||||
|
func NormalizeSessionID(raw string) (string, error) {
|
||||||
|
normalized := strings.TrimSpace(raw)
|
||||||
|
if normalized == "" {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
if length := utf8.RuneCountInString(normalized); length > SessionIDMaxLength {
|
||||||
|
return "", fmt.Errorf("session_id length %d exceeds maximum %d", length, SessionIDMaxLength)
|
||||||
|
}
|
||||||
|
return normalized, nil
|
||||||
|
}
|
||||||
57
internal/domain/session_test.go
Normal file
57
internal/domain/session_test.go
Normal file
@@ -0,0 +1,57 @@
|
|||||||
|
package domain
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNormalizeSessionID(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
raw string
|
||||||
|
want string
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "trims surrounding Unicode whitespace",
|
||||||
|
raw: "\u2003 session-123 \u2003",
|
||||||
|
want: "session-123",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "blank input is omitted",
|
||||||
|
raw: " \t\u2003 ",
|
||||||
|
want: "",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "maximum Unicode length is accepted",
|
||||||
|
raw: strings.Repeat("界", SessionIDMaxLength),
|
||||||
|
want: strings.Repeat("界", SessionIDMaxLength),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "one Unicode code point over maximum is rejected",
|
||||||
|
raw: strings.Repeat("界", SessionIDMaxLength+1),
|
||||||
|
wantErr: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
got, err := NormalizeSessionID(tt.raw)
|
||||||
|
if tt.wantErr {
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected normalization error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "exceeds maximum") {
|
||||||
|
t.Fatalf("expected useful length diagnostic, got %v", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("normalize session id: %v", err)
|
||||||
|
}
|
||||||
|
if got != tt.want {
|
||||||
|
t.Fatalf("normalized session id = %q, want %q", got, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,25 +1,30 @@
|
|||||||
package promptkit
|
// Package jsonvalue validates and defensively copies JSON-compatible value
|
||||||
|
// trees used by public configuration and request boundaries.
|
||||||
|
package jsonvalue
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"math"
|
"math"
|
||||||
"reflect"
|
"reflect"
|
||||||
|
"sort"
|
||||||
"strconv"
|
"strconv"
|
||||||
)
|
)
|
||||||
|
|
||||||
const maxSafeJSONInteger = 1<<53 - 1
|
const maxSafeJSONInteger = 1<<53 - 1
|
||||||
|
|
||||||
type jsonVisit struct {
|
type visit struct {
|
||||||
typ reflect.Type
|
typ reflect.Type
|
||||||
ptr uintptr
|
ptr uintptr
|
||||||
}
|
}
|
||||||
|
|
||||||
func copyPublicJSONMap(src map[string]any) (map[string]any, error) {
|
// CopyMap validates and deeply copies an extra-parameter map while preserving
|
||||||
|
// compatible concrete map, slice, array, scalar, and number types.
|
||||||
|
func CopyMap(src map[string]any) (map[string]any, error) {
|
||||||
if src == nil {
|
if src == nil {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
copied, err := copyPublicJSONValue(reflect.ValueOf(src), "extra_params", make(map[jsonVisit]struct{}))
|
copied, err := copyValue(reflect.ValueOf(src), "extra_params", make(map[visit]struct{}))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@@ -30,7 +35,7 @@ func copyPublicJSONMap(src map[string]any) (map[string]any, error) {
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func copyPublicJSONValue(value reflect.Value, path string, seen map[jsonVisit]struct{}) (any, error) {
|
func copyValue(value reflect.Value, path string, seen map[visit]struct{}) (any, error) {
|
||||||
if !value.IsValid() {
|
if !value.IsValid() {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
@@ -38,12 +43,15 @@ func copyPublicJSONValue(value reflect.Value, path string, seen map[jsonVisit]st
|
|||||||
if value.IsNil() {
|
if value.IsNil() {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
return copyPublicJSONValue(value.Elem(), path, seen)
|
return copyValue(value.Elem(), path, seen)
|
||||||
}
|
}
|
||||||
if !value.CanInterface() {
|
if !value.CanInterface() {
|
||||||
return nil, fmt.Errorf("%s: value cannot be copied", path)
|
return nil, fmt.Errorf("%s: value cannot be copied", path)
|
||||||
}
|
}
|
||||||
if number, ok := value.Interface().(json.Number); ok {
|
if number, ok := value.Interface().(json.Number); ok {
|
||||||
|
if _, err := json.Marshal(number); err != nil {
|
||||||
|
return nil, fmt.Errorf("%s: invalid JSON number", path)
|
||||||
|
}
|
||||||
f, err := strconv.ParseFloat(number.String(), 64)
|
f, err := strconv.ParseFloat(number.String(), 64)
|
||||||
if err != nil || math.IsNaN(f) || math.IsInf(f, 0) {
|
if err != nil || math.IsNaN(f) || math.IsInf(f, 0) {
|
||||||
return nil, fmt.Errorf("%s: invalid JSON number", path)
|
return nil, fmt.Errorf("%s: invalid JSON number", path)
|
||||||
@@ -65,8 +73,8 @@ func copyPublicJSONValue(value reflect.Value, path string, seen map[jsonVisit]st
|
|||||||
}
|
}
|
||||||
return value.Interface(), nil
|
return value.Interface(), nil
|
||||||
case reflect.Float32, reflect.Float64:
|
case reflect.Float32, reflect.Float64:
|
||||||
f := value.Convert(reflect.TypeOf(float64(0))).Float()
|
number := value.Convert(reflect.TypeOf(float64(0))).Float()
|
||||||
if math.IsNaN(f) || math.IsInf(f, 0) {
|
if math.IsNaN(number) || math.IsInf(number, 0) {
|
||||||
return nil, fmt.Errorf("%s: floating-point value must be finite", path)
|
return nil, fmt.Errorf("%s: floating-point value must be finite", path)
|
||||||
}
|
}
|
||||||
return value.Interface(), nil
|
return value.Interface(), nil
|
||||||
@@ -74,28 +82,28 @@ func copyPublicJSONValue(value reflect.Value, path string, seen map[jsonVisit]st
|
|||||||
if value.IsNil() {
|
if value.IsNil() {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
visit := jsonVisit{typ: value.Type(), ptr: value.Pointer()}
|
current := visit{typ: value.Type(), ptr: value.Pointer()}
|
||||||
if _, ok := seen[visit]; ok {
|
if _, ok := seen[current]; ok {
|
||||||
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
|
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
|
||||||
}
|
}
|
||||||
seen[visit] = struct{}{}
|
seen[current] = struct{}{}
|
||||||
defer delete(seen, visit)
|
defer delete(seen, current)
|
||||||
return copyPublicJSONValue(value.Elem(), path, seen)
|
return copyValue(value.Elem(), path, seen)
|
||||||
case reflect.Map:
|
case reflect.Map:
|
||||||
return copyPublicJSONMapValue(value, path, seen)
|
return copyMapValue(value, path, seen)
|
||||||
case reflect.Slice:
|
case reflect.Slice:
|
||||||
if value.IsNil() {
|
if value.IsNil() {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
return copyPublicJSONSequenceValue(value, path, seen)
|
return copySequenceValue(value, path, seen)
|
||||||
case reflect.Array:
|
case reflect.Array:
|
||||||
return copyPublicJSONSequenceValue(value, path, seen)
|
return copySequenceValue(value, path, seen)
|
||||||
default:
|
default:
|
||||||
return nil, fmt.Errorf("%s: unsupported JSON value type %s", path, value.Type())
|
return nil, fmt.Errorf("%s: unsupported JSON value type %s", path, value.Type())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func copyPublicJSONMapValue(value reflect.Value, path string, seen map[jsonVisit]struct{}) (any, error) {
|
func copyMapValue(value reflect.Value, path string, seen map[visit]struct{}) (any, error) {
|
||||||
if value.IsNil() {
|
if value.IsNil() {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
@@ -103,37 +111,43 @@ func copyPublicJSONMapValue(value reflect.Value, path string, seen map[jsonVisit
|
|||||||
return nil, fmt.Errorf("%s: map key type %s is not supported", path, value.Type().Key())
|
return nil, fmt.Errorf("%s: map key type %s is not supported", path, value.Type().Key())
|
||||||
}
|
}
|
||||||
|
|
||||||
visit := jsonVisit{typ: value.Type(), ptr: value.Pointer()}
|
current := visit{typ: value.Type(), ptr: value.Pointer()}
|
||||||
if _, ok := seen[visit]; ok {
|
if _, ok := seen[current]; ok {
|
||||||
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
|
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
|
||||||
}
|
}
|
||||||
seen[visit] = struct{}{}
|
seen[current] = struct{}{}
|
||||||
defer delete(seen, visit)
|
defer delete(seen, current)
|
||||||
|
|
||||||
|
keys := value.MapKeys()
|
||||||
|
sort.Slice(keys, func(i, j int) bool {
|
||||||
|
return keys[i].String() < keys[j].String()
|
||||||
|
})
|
||||||
|
|
||||||
type entry struct {
|
type entry struct {
|
||||||
key reflect.Value
|
key reflect.Value
|
||||||
name string
|
name string
|
||||||
value any
|
value any
|
||||||
}
|
}
|
||||||
entries := make([]entry, 0, value.Len())
|
entries := make([]entry, 0, len(keys))
|
||||||
preserveType := true
|
preserveType := true
|
||||||
elemType := value.Type().Elem()
|
elementType := value.Type().Elem()
|
||||||
iter := value.MapRange()
|
for _, key := range keys {
|
||||||
for iter.Next() {
|
|
||||||
key := iter.Key()
|
|
||||||
name := key.String()
|
name := key.String()
|
||||||
copied, err := copyPublicJSONValue(iter.Value(), path+"."+name, seen)
|
if name == "" {
|
||||||
|
return nil, fmt.Errorf("%s: map key must not be empty", path)
|
||||||
|
}
|
||||||
|
copied, err := copyValue(value.MapIndex(key), path+"."+name, seen)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
entries = append(entries, entry{key: key, name: name, value: copied})
|
entries = append(entries, entry{key: key, name: name, value: copied})
|
||||||
if copied == nil {
|
if copied == nil {
|
||||||
if !canAssignNil(elemType) {
|
if !canAssignNil(elementType) {
|
||||||
preserveType = false
|
preserveType = false
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if !reflect.TypeOf(copied).AssignableTo(elemType) {
|
if !reflect.TypeOf(copied).AssignableTo(elementType) {
|
||||||
preserveType = false
|
preserveType = false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -142,7 +156,7 @@ func copyPublicJSONMapValue(value reflect.Value, path string, seen map[jsonVisit
|
|||||||
out := reflect.MakeMapWithSize(value.Type(), len(entries))
|
out := reflect.MakeMapWithSize(value.Type(), len(entries))
|
||||||
for _, entry := range entries {
|
for _, entry := range entries {
|
||||||
if entry.value == nil {
|
if entry.value == nil {
|
||||||
out.SetMapIndex(entry.key, reflect.Zero(elemType))
|
out.SetMapIndex(entry.key, reflect.Zero(elementType))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
out.SetMapIndex(entry.key, reflect.ValueOf(entry.value))
|
out.SetMapIndex(entry.key, reflect.ValueOf(entry.value))
|
||||||
@@ -157,33 +171,33 @@ func copyPublicJSONMapValue(value reflect.Value, path string, seen map[jsonVisit
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func copyPublicJSONSequenceValue(value reflect.Value, path string, seen map[jsonVisit]struct{}) (any, error) {
|
func copySequenceValue(value reflect.Value, path string, seen map[visit]struct{}) (any, error) {
|
||||||
var visit jsonVisit
|
var current visit
|
||||||
if value.Kind() == reflect.Slice {
|
if value.Kind() == reflect.Slice {
|
||||||
visit = jsonVisit{typ: value.Type(), ptr: value.Pointer()}
|
current = visit{typ: value.Type(), ptr: value.Pointer()}
|
||||||
if _, ok := seen[visit]; ok {
|
if _, ok := seen[current]; ok {
|
||||||
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
|
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
|
||||||
}
|
}
|
||||||
seen[visit] = struct{}{}
|
seen[current] = struct{}{}
|
||||||
defer delete(seen, visit)
|
defer delete(seen, current)
|
||||||
}
|
}
|
||||||
|
|
||||||
values := make([]any, value.Len())
|
values := make([]any, value.Len())
|
||||||
preserveType := true
|
preserveType := true
|
||||||
elemType := value.Type().Elem()
|
elementType := value.Type().Elem()
|
||||||
for i := 0; i < value.Len(); i++ {
|
for i := 0; i < value.Len(); i++ {
|
||||||
copied, err := copyPublicJSONValue(value.Index(i), fmt.Sprintf("%s[%d]", path, i), seen)
|
copied, err := copyValue(value.Index(i), fmt.Sprintf("%s[%d]", path, i), seen)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
values[i] = copied
|
values[i] = copied
|
||||||
if copied == nil {
|
if copied == nil {
|
||||||
if !canAssignNil(elemType) {
|
if !canAssignNil(elementType) {
|
||||||
preserveType = false
|
preserveType = false
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
if !reflect.TypeOf(copied).AssignableTo(elemType) {
|
if !reflect.TypeOf(copied).AssignableTo(elementType) {
|
||||||
preserveType = false
|
preserveType = false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -195,7 +209,7 @@ func copyPublicJSONSequenceValue(value reflect.Value, path string, seen map[json
|
|||||||
}
|
}
|
||||||
for i, copied := range values {
|
for i, copied := range values {
|
||||||
if copied == nil {
|
if copied == nil {
|
||||||
out.Index(i).Set(reflect.Zero(elemType))
|
out.Index(i).Set(reflect.Zero(elementType))
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
out.Index(i).Set(reflect.ValueOf(copied))
|
out.Index(i).Set(reflect.ValueOf(copied))
|
||||||
97
internal/jsonvalue/jsonvalue_test.go
Normal file
97
internal/jsonvalue/jsonvalue_test.go
Normal file
@@ -0,0 +1,97 @@
|
|||||||
|
package jsonvalue_test
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"math"
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestCopyMapPreservesTypesAndIsolatesMutations(t *testing.T) {
|
||||||
|
nested := map[string]int{"limit": 2}
|
||||||
|
sequence := []string{"one", "two"}
|
||||||
|
input := map[string]any{
|
||||||
|
"count": int64(7),
|
||||||
|
"number": json.Number("-1.25e+2"),
|
||||||
|
"nested": nested,
|
||||||
|
"sequence": sequence,
|
||||||
|
}
|
||||||
|
|
||||||
|
copied, err := jsonvalue.CopyMap(input)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("copy map: %v", err)
|
||||||
|
}
|
||||||
|
nested["limit"] = 99
|
||||||
|
sequence[0] = "changed"
|
||||||
|
input["added"] = true
|
||||||
|
|
||||||
|
if got, ok := copied["count"].(int64); !ok || got != 7 {
|
||||||
|
t.Fatalf("integer type or value changed: %#v", copied["count"])
|
||||||
|
}
|
||||||
|
if got, ok := copied["number"].(json.Number); !ok || got != "-1.25e+2" {
|
||||||
|
t.Fatalf("JSON number type or value changed: %#v", copied["number"])
|
||||||
|
}
|
||||||
|
if got := copied["nested"].(map[string]int)["limit"]; got != 2 {
|
||||||
|
t.Fatalf("nested map was not isolated: %d", got)
|
||||||
|
}
|
||||||
|
if got := copied["sequence"].([]string)[0]; got != "one" {
|
||||||
|
t.Fatalf("sequence was not isolated: %q", got)
|
||||||
|
}
|
||||||
|
if _, ok := copied["added"]; ok {
|
||||||
|
t.Fatalf("top-level map was not isolated: %#v", copied)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCopyMapRejectsInvalidValues(t *testing.T) {
|
||||||
|
cyclicMap := map[string]any{}
|
||||||
|
cyclicMap["self"] = cyclicMap
|
||||||
|
cyclicSlice := []any{nil}
|
||||||
|
cyclicSlice[0] = cyclicSlice
|
||||||
|
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
value any
|
||||||
|
}{
|
||||||
|
{name: "empty nested key", value: map[string]int{"": 1}},
|
||||||
|
{name: "non-string map key", value: map[int]string{1: "one"}},
|
||||||
|
{name: "unsupported value", value: make(chan int)},
|
||||||
|
{name: "cyclic map", value: cyclicMap},
|
||||||
|
{name: "cyclic slice", value: cyclicSlice},
|
||||||
|
{name: "NaN", value: math.NaN()},
|
||||||
|
{name: "positive infinity", value: math.Inf(1)},
|
||||||
|
{name: "unsafe signed integer", value: int64(1 << 53)},
|
||||||
|
{name: "unsafe unsigned integer", value: uint64(1 << 53)},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tc := range tests {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
if _, err := jsonvalue.CopyMap(map[string]any{"value": tc.value}); err == nil {
|
||||||
|
t.Fatal("expected validation error")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCopyMapValidatesJSONNumberSyntaxAndRange(t *testing.T) {
|
||||||
|
for _, number := range []json.Number{"0", "-1", "1.25", "-1.25e+2"} {
|
||||||
|
t.Run("valid "+number.String(), func(t *testing.T) {
|
||||||
|
got, err := jsonvalue.CopyMap(map[string]any{"value": number})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("copy valid JSON number: %v", err)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got["value"], number) {
|
||||||
|
t.Fatalf("JSON number changed: got %#v want %#v", got["value"], number)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, number := range []json.Number{"", "01", "+1", "1.", ".1", "1e9999", "not-a-number"} {
|
||||||
|
t.Run("invalid "+number.String(), func(t *testing.T) {
|
||||||
|
if _, err := jsonvalue.CopyMap(map[string]any{"value": number}); err == nil {
|
||||||
|
t.Fatal("expected invalid JSON number error")
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -12,7 +12,6 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
"time"
|
"time"
|
||||||
"unicode/utf8"
|
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
@@ -177,12 +176,11 @@ func openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultMod
|
|||||||
wireReq := openAIChatRequest{
|
wireReq := openAIChatRequest{
|
||||||
Model: model,
|
Model: model,
|
||||||
}
|
}
|
||||||
if sessionID := strings.TrimSpace(req.Prompt.SessionID); sessionID != "" {
|
sessionID, err := domain.NormalizeSessionID(req.Prompt.SessionID)
|
||||||
if n := utf8.RuneCountInString(sessionID); n > domain.SessionIDMaxLength {
|
if err != nil {
|
||||||
return openAIChatRequest{}, fmt.Errorf("session_id length %d exceeds maximum %d", n, domain.SessionIDMaxLength)
|
return openAIChatRequest{}, err
|
||||||
}
|
}
|
||||||
wireReq.SessionID = sessionID
|
wireReq.SessionID = sessionID
|
||||||
}
|
|
||||||
|
|
||||||
wireReq.Messages = make([]openAIChatRequestMessage, 0, len(req.Prompt.Messages))
|
wireReq.Messages = make([]openAIChatRequestMessage, 0, len(req.Prompt.Messages))
|
||||||
for _, msg := range req.Prompt.Messages {
|
for _, msg := range req.Prompt.Messages {
|
||||||
@@ -262,7 +260,7 @@ func openAIChatRequestPayload(req openAIChatRequest) (map[string]any, error) {
|
|||||||
if key == "" {
|
if key == "" {
|
||||||
return nil, errors.New("extra_params key must not be empty")
|
return nil, errors.New("extra_params key must not be empty")
|
||||||
}
|
}
|
||||||
if _, reserved := reservedOpenAIChatRequestFields[key]; reserved {
|
if IsReservedOpenAIChatRequestField(key) {
|
||||||
return nil, fmt.Errorf("extra_params key %q collides with reserved request field", key)
|
return nil, fmt.Errorf("extra_params key %q collides with reserved request field", key)
|
||||||
}
|
}
|
||||||
if _, err := json.Marshal(value); err != nil {
|
if _, err := json.Marshal(value); err != nil {
|
||||||
@@ -274,16 +272,23 @@ func openAIChatRequestPayload(req openAIChatRequest) (map[string]any, error) {
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var reservedOpenAIChatRequestFields = map[string]struct{}{
|
// IsReservedOpenAIChatRequestField reports whether name is owned by the
|
||||||
"model": {},
|
// standard OpenAI-compatible chat request rather than extra parameters.
|
||||||
"session_id": {},
|
func IsReservedOpenAIChatRequestField(name string) bool {
|
||||||
"messages": {},
|
switch name {
|
||||||
"temperature": {},
|
case "model",
|
||||||
"max_tokens": {},
|
"session_id",
|
||||||
"top_p": {},
|
"messages",
|
||||||
"service_tier": {},
|
"temperature",
|
||||||
"reasoning_effort": {},
|
"max_tokens",
|
||||||
"response_format": {},
|
"top_p",
|
||||||
|
"service_tier",
|
||||||
|
"reasoning_effort",
|
||||||
|
"response_format":
|
||||||
|
return true
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
type openAIChatRequestMessage struct {
|
type openAIChatRequestMessage struct {
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
id: aion-2
|
id: aion-2
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: aion-labs/aion-2.0
|
model: aion-labs/aion-2.0
|
||||||
temperature: 0.72
|
temperature: 0.72
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
top_p: 0.95
|
top_p: 0.95
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: claude-fable-latest
|
id: claude-fable-latest
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "~anthropic/claude-fable-latest"
|
model: "~anthropic/claude-fable-latest"
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
timeout_seconds: 600
|
timeout_seconds: 600
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: claude-haiku-latest
|
id: claude-haiku-latest
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "~anthropic/claude-haiku-latest"
|
model: "~anthropic/claude-haiku-latest"
|
||||||
reasoning_effort: medium
|
reasoning_effort: medium
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: claude-opus-latest
|
id: claude-opus-latest
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "~anthropic/claude-opus-latest"
|
model: "~anthropic/claude-opus-latest"
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: claude-sonnet-latest
|
id: claude-sonnet-latest
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "~anthropic/claude-sonnet-latest"
|
model: "~anthropic/claude-sonnet-latest"
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: deepseek-3-2
|
id: deepseek-3-2
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: deepseek/deepseek-v3.2
|
model: deepseek/deepseek-v3.2
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: deepseek-4-flash
|
id: deepseek-4-flash
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: deepseek/deepseek-v4-flash
|
model: deepseek/deepseek-v4-flash
|
||||||
#reasoning_effort: medium
|
#reasoning_effort: medium
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: deepseek-4-pro
|
id: deepseek-4-pro
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: deepseek/deepseek-v4-pro
|
model: deepseek/deepseek-v4-pro
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
id: gemini-2-flash-lite
|
id: gemini-2-flash-lite
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "google/gemini-2.5-flash-lite"
|
model: "google/gemini-2.5-flash-lite"
|
||||||
#temperature: 0.15
|
#temperature: 0.15
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
#top_p: 0.98
|
#top_p: 0.98
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
id: gemini-2-flash
|
id: gemini-2-flash
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "google/gemini-2.5-flash"
|
model: "google/gemini-2.5-flash"
|
||||||
#temperature: 0.15
|
#temperature: 0.15
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
#top_p: 0.98
|
#top_p: 0.98
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
id: gemini-2-pro
|
id: gemini-2-pro
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "google/gemini-2.5-pro"
|
model: "google/gemini-2.5-pro"
|
||||||
#temperature: 0.15
|
#temperature: 0.15
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
#top_p: 0.98
|
#top_p: 0.98
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
id: gemini-3-flash-lite
|
id: gemini-3-flash-lite
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "google/gemini-3.1-flash-lite"
|
model: "google/gemini-3.1-flash-lite"
|
||||||
#temperature: 0.15
|
#temperature: 0.15
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
#top_p: 0.98
|
#top_p: 0.98
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
id: gemini-flash-latest
|
id: gemini-flash-latest
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "~google/gemini-flash-latest"
|
model: "~google/gemini-flash-latest"
|
||||||
#temperature: 0.15
|
#temperature: 0.15
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
#top_p: 0.98
|
#top_p: 0.98
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
id: gemini-pro-latest
|
id: gemini-pro-latest
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "~google/gemini-pro-latest"
|
model: "~google/gemini-pro-latest"
|
||||||
#temperature: 0.15
|
#temperature: 0.15
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
#top_p: 0.98
|
#top_p: 0.98
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
id: gemma-4-31b
|
id: gemma-4-31b
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: google/gemma-4-31b-it:exacto
|
model: google/gemma-4-31b-it:exacto
|
||||||
temperature: 0.15
|
temperature: 0.15
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
top_p: 0.98
|
top_p: 0.98
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
id: minimax-m2
|
id: minimax-m2
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: minimax/minimax-m2.5
|
model: minimax/minimax-m2.5
|
||||||
temperature: 0.5
|
temperature: 0.5
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
top_p: 0.95
|
top_p: 0.95
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
id: minimax-m3
|
id: minimax-m3
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: minimax/minimax-m3
|
model: minimax/minimax-m3
|
||||||
#temperature: 0.5
|
#temperature: 0.5
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
#top_p: 0.95
|
#top_p: 0.95
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: mistral-large-2512
|
id: mistral-large-2512
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: mistralai/mistral-large-2512
|
model: mistralai/mistral-large-2512
|
||||||
temperature: 0.15
|
temperature: 0.15
|
||||||
top_p: 0.98
|
top_p: 0.98
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
|
|||||||
@@ -1,8 +1,7 @@
|
|||||||
id: mistral-medium-3-5
|
id: mistral-medium-3-5
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: mistralai/mistral-medium-3-5
|
model: mistralai/mistral-medium-3-5
|
||||||
temperature: 0.15
|
temperature: 0.15
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
top_p: 0.98
|
top_p: 0.98
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: mistral-small-3
|
id: mistral-small-3
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: mistralai/mistral-small-3.2-24b-instruct
|
model: mistralai/mistral-small-3.2-24b-instruct
|
||||||
temperature: 0.05
|
temperature: 0.05
|
||||||
top_p: 1.0
|
top_p: 1.0
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
|
|||||||
@@ -1,8 +1,7 @@
|
|||||||
id: mistral-small-4
|
id: mistral-small-4
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: mistralai/mistral-small-2603
|
model: mistralai/mistral-small-2603
|
||||||
temperature: 0.1
|
temperature: 0.1
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
top_p: 0.98
|
top_p: 0.98
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: nemotron-3-ultra
|
id: nemotron-3-ultra
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: nvidia/nemotron-3-ultra-550b-a55b
|
model: nvidia/nemotron-3-ultra-550b-a55b
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: gpt-5-mini
|
id: gpt-5-mini
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "openai/gpt-5.4-mini"
|
model: "openai/gpt-5.4-mini"
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: gpt-5-nano
|
id: gpt-5-nano
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "openai/gpt-5.4-nano"
|
model: "openai/gpt-5.4-nano"
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
@@ -28,6 +29,12 @@ func TestBuiltInProfilesValidateThroughRepository(t *testing.T) {
|
|||||||
if p.ID != id {
|
if p.ID != id {
|
||||||
t.Fatalf("expected profile id %q, got %q", id, p.ID)
|
t.Fatalf("expected profile id %q, got %q", id, p.ID)
|
||||||
}
|
}
|
||||||
|
if p.BackendID != backend.OpenRouterID {
|
||||||
|
t.Fatalf("expected profile %q to select %q, got %q", id, backend.OpenRouterID, p.BackendID)
|
||||||
|
}
|
||||||
|
if p.Endpoint != "" || p.APIKeyEnv != "" {
|
||||||
|
t.Fatalf("expected profile %q to inherit backend connection settings, got endpoint=%q api_key_env=%q", id, p.Endpoint, p.APIKeyEnv)
|
||||||
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -60,6 +67,15 @@ func loadBuiltInProfileIDs(t *testing.T) map[string]string {
|
|||||||
if _, ok := raw["api_key"]; ok {
|
if _, ok := raw["api_key"]; ok {
|
||||||
t.Fatalf("built-in profile %s contains raw api_key", name)
|
t.Fatalf("built-in profile %s contains raw api_key", name)
|
||||||
}
|
}
|
||||||
|
if raw["backend"] != backend.OpenRouterID {
|
||||||
|
t.Fatalf("built-in profile %s does not select %q", name, backend.OpenRouterID)
|
||||||
|
}
|
||||||
|
if _, ok := raw["endpoint"]; ok {
|
||||||
|
t.Fatalf("built-in profile %s repeats endpoint", name)
|
||||||
|
}
|
||||||
|
if _, ok := raw["api_key_env"]; ok {
|
||||||
|
t.Fatalf("built-in profile %s repeats api_key_env", name)
|
||||||
|
}
|
||||||
id, ok := raw["id"].(string)
|
id, ok := raw["id"].(string)
|
||||||
if !ok || strings.TrimSpace(id) == "" {
|
if !ok || strings.TrimSpace(id) == "" {
|
||||||
t.Fatalf("built-in profile %s has missing id", name)
|
t.Fatalf("built-in profile %s has missing id", name)
|
||||||
|
|||||||
@@ -121,6 +121,7 @@ func loadProfile(ctx context.Context, fsys fs.FS, root string, id string) (*doma
|
|||||||
if prof.ID != id {
|
if prof.ID != id {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
prof.BackendID = strings.TrimSpace(prof.BackendID)
|
||||||
if err := validateProfile(&prof); err != nil {
|
if err := validateProfile(&prof); err != nil {
|
||||||
if errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
if errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
||||||
return nil, fmt.Errorf("%w: %s", err, relPath)
|
return nil, fmt.Errorf("%w: %s", err, relPath)
|
||||||
@@ -189,8 +190,8 @@ func validateProfile(p *domain.ExecutionProfile) error {
|
|||||||
if strings.TrimSpace(p.ID) == "" {
|
if strings.TrimSpace(p.ID) == "" {
|
||||||
return errors.New("id is required")
|
return errors.New("id is required")
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(p.Endpoint) == "" {
|
if strings.TrimSpace(p.BackendID) == "" && strings.TrimSpace(p.Endpoint) == "" {
|
||||||
return errors.New("endpoint is required")
|
return errors.New("backend or endpoint is required")
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(p.Model) == "" {
|
if strings.TrimSpace(p.Model) == "" {
|
||||||
return errors.New("model is required")
|
return errors.New("model is required")
|
||||||
|
|||||||
@@ -52,6 +52,43 @@ func TestFilesystemRepository_GetProfile(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("backend and endpoint connection matrix", func(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
connection string
|
||||||
|
wantBackend string
|
||||||
|
wantEndpoint string
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{name: "backend only", connection: "backend: ' openrouter '", wantBackend: "openrouter"},
|
||||||
|
{name: "endpoint only", connection: "endpoint: http://localhost:8000/v1", wantEndpoint: "http://localhost:8000/v1"},
|
||||||
|
{name: "both", connection: "backend: openrouter\nendpoint: http://localhost:8000/v1", wantBackend: "openrouter", wantEndpoint: "http://localhost:8000/v1"},
|
||||||
|
{name: "neither", wantErr: true},
|
||||||
|
{name: "blank backend", connection: "backend: ' '", wantErr: true},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
id := "connection-" + strings.ReplaceAll(tt.name, " ", "-")
|
||||||
|
writeProfileTestFile(t, filepath.Join(tmpDir, id+".yaml"), "id: "+id+"\nmodel: model\n"+tt.connection+"\n")
|
||||||
|
|
||||||
|
p, err := repo.GetProfile(ctx, id)
|
||||||
|
if tt.wantErr {
|
||||||
|
if !errors.Is(err, ErrInvalidProfile) {
|
||||||
|
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected profile to load, got %v", err)
|
||||||
|
}
|
||||||
|
if p.BackendID != tt.wantBackend || p.Endpoint != tt.wantEndpoint {
|
||||||
|
t.Fatalf("unexpected connection values: backend=%q endpoint=%q", p.BackendID, p.Endpoint)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("valid profile with api_key_env", func(t *testing.T) {
|
t.Run("valid profile with api_key_env", func(t *testing.T) {
|
||||||
p, err := repo.GetProfile(ctx, "local-secure")
|
p, err := repo.GetProfile(ctx, "local-secure")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -5,10 +5,9 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
|
||||||
"strings"
|
|
||||||
"text/template"
|
"text/template"
|
||||||
"unicode/utf8"
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -95,10 +94,6 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
|
|||||||
}
|
}
|
||||||
|
|
||||||
func renderSessionID(raw string, funcs template.FuncMap, vars map[string]string) (string, error) {
|
func renderSessionID(raw string, funcs template.FuncMap, vars map[string]string) (string, error) {
|
||||||
if strings.TrimSpace(raw) == "" {
|
|
||||||
return "", nil
|
|
||||||
}
|
|
||||||
|
|
||||||
tmpl, err := template.New("session_id").Funcs(funcs).Option("missingkey=error").Parse(raw)
|
tmpl, err := template.New("session_id").Funcs(funcs).Option("missingkey=error").Parse(raw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", fmt.Errorf("%w: session_id: %v", ErrInvalidTemplate, err)
|
return "", fmt.Errorf("%w: session_id: %v", ErrInvalidTemplate, err)
|
||||||
@@ -109,9 +104,9 @@ func renderSessionID(raw string, funcs template.FuncMap, vars map[string]string)
|
|||||||
return "", fmt.Errorf("%w: session_id: %w", ErrRenderFailure, err)
|
return "", fmt.Errorf("%w: session_id: %w", ErrRenderFailure, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
sessionID := strings.TrimSpace(buf.String())
|
sessionID, err := domain.NormalizeSessionID(buf.String())
|
||||||
if n := utf8.RuneCountInString(sessionID); n > domain.SessionIDMaxLength {
|
if err != nil {
|
||||||
return "", fmt.Errorf("%w: session_id length %d exceeds maximum %d", ErrRenderFailure, n, domain.SessionIDMaxLength)
|
return "", fmt.Errorf("%w: session_id: %v", ErrRenderFailure, err)
|
||||||
}
|
}
|
||||||
return sessionID, nil
|
return sessionID, nil
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,6 +17,7 @@ type OutputRepairer interface {
|
|||||||
type RepairRequest struct {
|
type RepairRequest struct {
|
||||||
PreviousOutput string
|
PreviousOutput string
|
||||||
ValidationErrors []string
|
ValidationErrors []string
|
||||||
|
SessionID string
|
||||||
Target domain.ExecutionTarget
|
Target domain.ExecutionTarget
|
||||||
StructuredOutput *domain.StructuredOutputSpec
|
StructuredOutput *domain.StructuredOutputSpec
|
||||||
Attempt int
|
Attempt int
|
||||||
@@ -42,7 +43,9 @@ func (r *defaultOutputRepairer) Repair(ctx context.Context, req RepairRequest) (
|
|||||||
errs = strings.Join(req.ValidationErrors, "\n")
|
errs = strings.Join(req.ValidationErrors, "\n")
|
||||||
}
|
}
|
||||||
|
|
||||||
prompt := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
prompt := domain.RenderedPrompt{
|
||||||
|
SessionID: req.SessionID,
|
||||||
|
Messages: []domain.RenderedMessage{
|
||||||
{
|
{
|
||||||
Role: "system",
|
Role: "system",
|
||||||
Content: "You repair invalid JSON output. Return only corrected JSON. Do not include explanations or markdown code fences.",
|
Content: "You repair invalid JSON output. Return only corrected JSON. Do not include explanations or markdown code fences.",
|
||||||
@@ -58,7 +61,8 @@ func (r *defaultOutputRepairer) Repair(ctx context.Context, req RepairRequest) (
|
|||||||
req.PreviousOutput,
|
req.PreviousOutput,
|
||||||
),
|
),
|
||||||
},
|
},
|
||||||
}}
|
},
|
||||||
|
}
|
||||||
|
|
||||||
resp, err := r.llm.Generate(ctx, domain.GenerateRequest{
|
resp, err := r.llm.Generate(ctx, domain.GenerateRequest{
|
||||||
Prompt: prompt,
|
Prompt: prompt,
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ var (
|
|||||||
type Runner struct {
|
type Runner struct {
|
||||||
promptDefs promptdef.Repository
|
promptDefs promptdef.Repository
|
||||||
profiles profile.Repository
|
profiles profile.Repository
|
||||||
|
backends BackendResolver
|
||||||
artifacts artifact.Reader
|
artifacts artifact.Reader
|
||||||
renderer prompt.Renderer
|
renderer prompt.Renderer
|
||||||
llm llm.Client
|
llm llm.Client
|
||||||
@@ -47,20 +48,27 @@ type Runner struct {
|
|||||||
repairer OutputRepairer
|
repairer OutputRepairer
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// BackendResolver resolves one normalized backend ID.
|
||||||
|
type BackendResolver interface {
|
||||||
|
GetBackend(string) (domain.Backend, error)
|
||||||
|
}
|
||||||
|
|
||||||
func NewRunner(
|
func NewRunner(
|
||||||
promptDefs promptdef.Repository,
|
promptDefs promptdef.Repository,
|
||||||
profiles profile.Repository,
|
profiles profile.Repository,
|
||||||
|
backends BackendResolver,
|
||||||
artifacts artifact.Reader,
|
artifacts artifact.Reader,
|
||||||
renderer prompt.Renderer,
|
renderer prompt.Renderer,
|
||||||
llmClient llm.Client,
|
llmClient llm.Client,
|
||||||
validator validate.Validator,
|
validator validate.Validator,
|
||||||
) *Runner {
|
) *Runner {
|
||||||
return NewRunnerWithRepairer(promptDefs, profiles, artifacts, renderer, llmClient, validator, nil)
|
return NewRunnerWithRepairer(promptDefs, profiles, backends, artifacts, renderer, llmClient, validator, nil)
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewRunnerWithRepairer(
|
func NewRunnerWithRepairer(
|
||||||
promptDefs promptdef.Repository,
|
promptDefs promptdef.Repository,
|
||||||
profiles profile.Repository,
|
profiles profile.Repository,
|
||||||
|
backends BackendResolver,
|
||||||
artifacts artifact.Reader,
|
artifacts artifact.Reader,
|
||||||
renderer prompt.Renderer,
|
renderer prompt.Renderer,
|
||||||
llmClient llm.Client,
|
llmClient llm.Client,
|
||||||
@@ -70,6 +78,7 @@ func NewRunnerWithRepairer(
|
|||||||
return &Runner{
|
return &Runner{
|
||||||
promptDefs: promptDefs,
|
promptDefs: promptDefs,
|
||||||
profiles: profiles,
|
profiles: profiles,
|
||||||
|
backends: backends,
|
||||||
artifacts: artifacts,
|
artifacts: artifacts,
|
||||||
renderer: renderer,
|
renderer: renderer,
|
||||||
llm: llmClient,
|
llm: llmClient,
|
||||||
@@ -118,6 +127,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
|||||||
repairResp, repairErr := r.repairer.Repair(ctx, RepairRequest{
|
repairResp, repairErr := r.repairer.Repair(ctx, RepairRequest{
|
||||||
PreviousOutput: genResp.Content,
|
PreviousOutput: genResp.Content,
|
||||||
ValidationErrors: validationResult.Errors,
|
ValidationErrors: validationResult.Errors,
|
||||||
|
SessionID: prepared.SessionID,
|
||||||
Target: prepared.EffectiveModelParams,
|
Target: prepared.EffectiveModelParams,
|
||||||
StructuredOutput: prepared.StructuredOutput,
|
StructuredOutput: prepared.StructuredOutput,
|
||||||
Attempt: attemptsUsed,
|
Attempt: attemptsUsed,
|
||||||
@@ -151,8 +161,10 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
|||||||
PromptID: prepared.PromptID,
|
PromptID: prepared.PromptID,
|
||||||
PromptVersion: prepared.PromptVersion,
|
PromptVersion: prepared.PromptVersion,
|
||||||
PromptHash: prepared.PromptHash,
|
PromptHash: prepared.PromptHash,
|
||||||
|
SessionID: prepared.SessionID,
|
||||||
RenderedPromptHash: prepared.RenderedPromptHash,
|
RenderedPromptHash: prepared.RenderedPromptHash,
|
||||||
SelectedProfileID: prepared.SelectedProfileID,
|
SelectedProfileID: prepared.SelectedProfileID,
|
||||||
|
SelectedBackendID: prepared.SelectedBackendID,
|
||||||
ModelName: prepared.EffectiveModelParams.Model,
|
ModelName: prepared.EffectiveModelParams.Model,
|
||||||
Endpoint: prepared.EffectiveModelParams.Endpoint,
|
Endpoint: prepared.EffectiveModelParams.Endpoint,
|
||||||
EffectiveModelParams: prepared.EffectiveModelParams,
|
EffectiveModelParams: prepared.EffectiveModelParams,
|
||||||
@@ -168,6 +180,10 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
|
|||||||
if strings.TrimSpace(req.PromptID) == "" {
|
if strings.TrimSpace(req.PromptID) == "" {
|
||||||
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
|
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
|
||||||
}
|
}
|
||||||
|
directSessionID, err := domain.NormalizeSessionID(req.SessionID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: session_id: %v", ErrInvalidRequest, err)
|
||||||
|
}
|
||||||
|
|
||||||
start := time.Now().UTC()
|
start := time.Now().UTC()
|
||||||
|
|
||||||
@@ -193,7 +209,20 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
|
|||||||
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
effectiveModel, targetPresence, err := resolveExecutionTarget(execProfile, req.Execution)
|
var selectedBackend *domain.Backend
|
||||||
|
if backendID := strings.TrimSpace(execProfile.BackendID); backendID != "" {
|
||||||
|
execProfile.BackendID = backendID
|
||||||
|
if r.backends == nil {
|
||||||
|
return nil, fmt.Errorf("%w: backend %q cannot be resolved", ErrProfileLoad, backendID)
|
||||||
|
}
|
||||||
|
resolvedBackend, resolveErr := r.backends.GetBackend(backendID)
|
||||||
|
if resolveErr != nil {
|
||||||
|
return nil, fmt.Errorf("%w: backend %q: %w", ErrProfileLoad, backendID, resolveErr)
|
||||||
|
}
|
||||||
|
selectedBackend = &resolvedBackend
|
||||||
|
}
|
||||||
|
|
||||||
|
effectiveModel, targetPresence, err := resolveExecutionTarget(selectedBackend, execProfile, req.Execution)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
@@ -228,10 +257,19 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
|
|||||||
inputHashes[name] = art.Hash
|
inputHashes[name] = art.Hash
|
||||||
}
|
}
|
||||||
|
|
||||||
renderedPrompt, err := r.renderer.Render(ctx, def, resolvedInputs, req.Vars)
|
definitionToRender := def
|
||||||
|
if directSessionID != "" {
|
||||||
|
definitionCopy := *def
|
||||||
|
definitionCopy.SessionID = ""
|
||||||
|
definitionToRender = &definitionCopy
|
||||||
|
}
|
||||||
|
renderedPrompt, err := r.renderer.Render(ctx, definitionToRender, resolvedInputs, req.Vars)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrPromptRender, err)
|
return nil, fmt.Errorf("%w: %w", ErrPromptRender, err)
|
||||||
}
|
}
|
||||||
|
if directSessionID != "" {
|
||||||
|
renderedPrompt.SessionID = directSessionID
|
||||||
|
}
|
||||||
|
|
||||||
end := time.Now().UTC()
|
end := time.Now().UTC()
|
||||||
return &domain.PreparedRun{
|
return &domain.PreparedRun{
|
||||||
@@ -239,6 +277,7 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
|
|||||||
PromptVersion: def.Version,
|
PromptVersion: def.Version,
|
||||||
PromptHash: promptDefinitionHash,
|
PromptHash: promptDefinitionHash,
|
||||||
SelectedProfileID: selectedProfileID,
|
SelectedProfileID: selectedProfileID,
|
||||||
|
SelectedBackendID: effectiveModel.BackendID,
|
||||||
EffectiveModelParams: effectiveModel,
|
EffectiveModelParams: effectiveModel,
|
||||||
TargetPresence: targetPresence,
|
TargetPresence: targetPresence,
|
||||||
OutputContract: effectiveContract,
|
OutputContract: effectiveContract,
|
||||||
@@ -338,7 +377,10 @@ func (r *Runner) shouldAttemptRepair(contract domain.OutputContract, validationR
|
|||||||
|
|
||||||
func mergeExecutionTarget(base domain.ExecutionTarget, override domain.ExecutionTarget) domain.ExecutionTarget {
|
func mergeExecutionTarget(base domain.ExecutionTarget, override domain.ExecutionTarget) domain.ExecutionTarget {
|
||||||
out := base
|
out := base
|
||||||
if override.Endpoint != "" {
|
if strings.TrimSpace(override.BackendID) != "" {
|
||||||
|
out.BackendID = override.BackendID
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(override.Endpoint) != "" {
|
||||||
out.Endpoint = override.Endpoint
|
out.Endpoint = override.Endpoint
|
||||||
}
|
}
|
||||||
if override.Model != "" {
|
if override.Model != "" {
|
||||||
@@ -367,6 +409,7 @@ func mergeExecutionTarget(base domain.ExecutionTarget, override domain.Execution
|
|||||||
}
|
}
|
||||||
if override.APIKeyRequired {
|
if override.APIKeyRequired {
|
||||||
out.APIKeyRequired = true
|
out.APIKeyRequired = true
|
||||||
|
out.APIKeyEnv = ""
|
||||||
}
|
}
|
||||||
if len(override.ExtraParams) > 0 {
|
if len(override.ExtraParams) > 0 {
|
||||||
out.ExtraParams = copyExtraParams(override.ExtraParams)
|
out.ExtraParams = copyExtraParams(override.ExtraParams)
|
||||||
@@ -414,8 +457,8 @@ func mergeExecutionTargetOverride(base domain.ExecutionTarget, override domain.E
|
|||||||
if strings.TrimSpace(override.ServiceTier) != "" {
|
if strings.TrimSpace(override.ServiceTier) != "" {
|
||||||
out.ServiceTier = override.ServiceTier
|
out.ServiceTier = override.ServiceTier
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(override.ReasoningEffort) != "" {
|
if override.ReasoningEffort != nil {
|
||||||
out.ReasoningEffort = override.ReasoningEffort
|
out.ReasoningEffort = strings.TrimSpace(*override.ReasoningEffort)
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(override.APIKeyEnv) != "" {
|
if strings.TrimSpace(override.APIKeyEnv) != "" {
|
||||||
out.APIKeyEnv = override.APIKeyEnv
|
out.APIKeyEnv = override.APIKeyEnv
|
||||||
@@ -426,8 +469,9 @@ func mergeExecutionTargetOverride(base domain.ExecutionTarget, override domain.E
|
|||||||
return out, presence, nil
|
return out, presence, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) {
|
func resolveExecutionTarget(backendValue *domain.Backend, profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) {
|
||||||
out := defaults.ExecutionTargetDefault()
|
out := defaults.ExecutionTargetDefault()
|
||||||
|
out = mergeExecutionTarget(out, backendToTarget(backendValue))
|
||||||
out = mergeExecutionTarget(out, executionProfileToTarget(profileValue))
|
out = mergeExecutionTarget(out, executionProfileToTarget(profileValue))
|
||||||
var presence domain.ExecutionTargetPresence
|
var presence domain.ExecutionTargetPresence
|
||||||
if override != nil {
|
if override != nil {
|
||||||
@@ -461,8 +505,13 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
|
|||||||
if p == nil {
|
if p == nil {
|
||||||
return domain.ExecutionTarget{}
|
return domain.ExecutionTarget{}
|
||||||
}
|
}
|
||||||
|
endpoint := p.Endpoint
|
||||||
|
if strings.TrimSpace(endpoint) == "" {
|
||||||
|
endpoint = ""
|
||||||
|
}
|
||||||
return domain.ExecutionTarget{
|
return domain.ExecutionTarget{
|
||||||
Endpoint: p.Endpoint,
|
BackendID: p.BackendID,
|
||||||
|
Endpoint: endpoint,
|
||||||
Model: p.Model,
|
Model: p.Model,
|
||||||
Temperature: p.Temperature,
|
Temperature: p.Temperature,
|
||||||
MaxTokens: p.MaxTokens,
|
MaxTokens: p.MaxTokens,
|
||||||
@@ -476,6 +525,18 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func backendToTarget(value *domain.Backend) domain.ExecutionTarget {
|
||||||
|
if value == nil {
|
||||||
|
return domain.ExecutionTarget{}
|
||||||
|
}
|
||||||
|
return domain.ExecutionTarget{
|
||||||
|
BackendID: value.ID,
|
||||||
|
Endpoint: value.Endpoint,
|
||||||
|
APIKeyEnv: value.APIKeyEnv,
|
||||||
|
ExtraParams: copyExtraParams(value.ExtraParams),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func copyExtraParams(src map[string]any) map[string]any {
|
func copyExtraParams(src map[string]any) map[string]any {
|
||||||
if len(src) == 0 {
|
if len(src) == 0 {
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -34,6 +34,18 @@ type fakeExecutionProfileRepo struct {
|
|||||||
lastID string
|
lastID string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type fakeBackendResolver struct {
|
||||||
|
backends map[string]domain.Backend
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f fakeBackendResolver) GetBackend(id string) (domain.Backend, error) {
|
||||||
|
value, ok := f.backends[id]
|
||||||
|
if !ok {
|
||||||
|
return domain.Backend{}, errors.New("backend not found")
|
||||||
|
}
|
||||||
|
return value, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (f *fakeExecutionProfileRepo) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
func (f *fakeExecutionProfileRepo) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||||
f.lastID = id
|
f.lastID = id
|
||||||
if f.err != nil {
|
if f.err != nil {
|
||||||
@@ -167,7 +179,7 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) {
|
|||||||
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
||||||
llmClient := &fakeLLM{forbid: true}
|
llmClient := &fakeLLM{forbid: true}
|
||||||
|
|
||||||
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil)
|
||||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
PromptVersion: "1",
|
PromptVersion: "1",
|
||||||
@@ -210,6 +222,145 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRunnerDirectSessionResolution(t *testing.T) {
|
||||||
|
t.Run("direct value wins and changes only the rendered prompt hash", func(t *testing.T) {
|
||||||
|
def := promptDef(domain.FormatText, domain.ValidationNone, 0)
|
||||||
|
def.SessionID = "template-{{.template_session}}"
|
||||||
|
promptRepo := &fakePromptRepo{def: def}
|
||||||
|
runner := NewRunner(
|
||||||
|
promptRepo,
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
nil,
|
||||||
|
defaultArtifactReader(),
|
||||||
|
prompt.NewGoRenderer(),
|
||||||
|
&fakeLLM{forbid: true},
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
req := domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
Vars: map[string]string{"template_session": "from-template"},
|
||||||
|
}
|
||||||
|
|
||||||
|
req.SessionID = " direct-one "
|
||||||
|
first, err := runner.Prepare(context.Background(), req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare first direct session: %v", err)
|
||||||
|
}
|
||||||
|
req.SessionID = "direct-two"
|
||||||
|
second, err := runner.Prepare(context.Background(), req)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare second direct session: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if first.SessionID != "direct-one" || second.SessionID != "direct-two" {
|
||||||
|
t.Fatalf("direct sessions were not normalized: first=%q second=%q", first.SessionID, second.SessionID)
|
||||||
|
}
|
||||||
|
if first.PromptHash != second.PromptHash {
|
||||||
|
t.Fatalf("direct session changed prompt-definition hash: first=%q second=%q", first.PromptHash, second.PromptHash)
|
||||||
|
}
|
||||||
|
if first.RenderedPromptHash == second.RenderedPromptHash {
|
||||||
|
t.Fatal("changing direct session did not change rendered-prompt hash")
|
||||||
|
}
|
||||||
|
if def.SessionID != "template-{{.template_session}}" {
|
||||||
|
t.Fatalf("repository-owned prompt definition was mutated: %q", def.SessionID)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("direct value bypasses failing session template without changing messages", func(t *testing.T) {
|
||||||
|
def := promptDef(domain.FormatText, domain.ValidationNone, 0)
|
||||||
|
def.SessionID = "{{.missing_session}}"
|
||||||
|
def.Templates = []domain.PromptMessageTemplate{
|
||||||
|
{Role: "user", Content: "Hello {{.name}}"},
|
||||||
|
}
|
||||||
|
runner := NewRunner(
|
||||||
|
&fakePromptRepo{def: def},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
nil,
|
||||||
|
defaultArtifactReader(),
|
||||||
|
prompt.NewGoRenderer(),
|
||||||
|
&fakeLLM{forbid: true},
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
|
||||||
|
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
SessionID: "direct-session",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
Vars: map[string]string{"name": "Rin"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare with direct session: %v", err)
|
||||||
|
}
|
||||||
|
if prepared.SessionID != "direct-session" {
|
||||||
|
t.Fatalf("prepared session id = %q, want direct-session", prepared.SessionID)
|
||||||
|
}
|
||||||
|
if len(prepared.Messages) != 1 || prepared.Messages[0].Content != "Hello Rin" {
|
||||||
|
t.Fatalf("message templates did not render normally: %+v", prepared.Messages)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("blank direct value retains prompt template behavior", func(t *testing.T) {
|
||||||
|
def := promptDef(domain.FormatText, domain.ValidationNone, 0)
|
||||||
|
def.SessionID = " template-{{.template_session}} "
|
||||||
|
runner := NewRunner(
|
||||||
|
&fakePromptRepo{def: def},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
nil,
|
||||||
|
defaultArtifactReader(),
|
||||||
|
prompt.NewGoRenderer(),
|
||||||
|
&fakeLLM{forbid: true},
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
|
||||||
|
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
SessionID: " \t ",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
Vars: map[string]string{"template_session": "rendered"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare with prompt session template: %v", err)
|
||||||
|
}
|
||||||
|
if prepared.SessionID != "template-rendered" {
|
||||||
|
t.Fatalf("prepared session id = %q, want template-rendered", prepared.SessionID)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("overlong direct value fails before loading or generation", func(t *testing.T) {
|
||||||
|
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
||||||
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "unexpected"}}
|
||||||
|
runner := NewRunner(
|
||||||
|
promptRepo,
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||||
|
nil,
|
||||||
|
defaultArtifactReader(),
|
||||||
|
defaultRenderer(),
|
||||||
|
llmClient,
|
||||||
|
nil,
|
||||||
|
)
|
||||||
|
|
||||||
|
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
SessionID: strings.Repeat("界", domain.SessionIDMaxLength+1),
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
})
|
||||||
|
if !errors.Is(err, ErrInvalidRequest) {
|
||||||
|
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
||||||
|
}
|
||||||
|
if promptRepo.lastID != "" {
|
||||||
|
t.Fatalf("invalid direct session loaded prompt %q", promptRepo.lastID)
|
||||||
|
}
|
||||||
|
if llmClient.calls != 0 {
|
||||||
|
t.Fatalf("invalid direct session invoked generation %d times", llmClient.calls)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func TestRunnerPrepareUsesPromptDefaultProfileWhenNoExplicitProfileID(t *testing.T) {
|
func TestRunnerPrepareUsesPromptDefaultProfileWhenNoExplicitProfileID(t *testing.T) {
|
||||||
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
||||||
promptRepo.def.DefaultProfile = "from-prompt"
|
promptRepo.def.DefaultProfile = "from-prompt"
|
||||||
@@ -257,7 +408,7 @@ func TestRunnerPrepareSelectedProfileDoesNotExistFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerPreparePromptLoadFailure(t *testing.T) {
|
func TestRunnerPreparePromptLoadFailure(t *testing.T) {
|
||||||
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
|
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
|
||||||
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p"})
|
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p"})
|
||||||
if !errors.Is(err, ErrPromptLoad) {
|
if !errors.Is(err, ErrPromptLoad) {
|
||||||
t.Fatalf("expected ErrPromptLoad, got %v", err)
|
t.Fatalf("expected ErrPromptLoad, got %v", err)
|
||||||
@@ -281,7 +432,7 @@ func TestRunnerPrepareRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) {
|
|||||||
ServiceTier: "priority",
|
ServiceTier: "priority",
|
||||||
},
|
},
|
||||||
}}
|
}}
|
||||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||||
|
|
||||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -379,12 +530,12 @@ func TestRunnerPrepareRequestNumericOverridePresence(t *testing.T) {
|
|||||||
TopP: 0.8,
|
TopP: 0.8,
|
||||||
TimeoutSeconds: 45,
|
TimeoutSeconds: 45,
|
||||||
},
|
},
|
||||||
}},
|
}}, nil,
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
defaultRenderer(),
|
defaultRenderer(),
|
||||||
&fakeLLM{forbid: true},
|
&fakeLLM{forbid: true},
|
||||||
nil,
|
nil)
|
||||||
)
|
|
||||||
|
|
||||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -426,12 +577,12 @@ func TestRunnerPrepareInvalidRequestNumericOverridesFail(t *testing.T) {
|
|||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
defaultRenderer(),
|
defaultRenderer(),
|
||||||
&fakeLLM{forbid: true},
|
&fakeLLM{forbid: true},
|
||||||
nil,
|
nil)
|
||||||
)
|
|
||||||
|
|
||||||
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -458,7 +609,7 @@ func TestRunnerPrepareSelectedProfileBeatsBuiltInDefault(t *testing.T) {
|
|||||||
ServiceTier: "priority",
|
ServiceTier: "priority",
|
||||||
},
|
},
|
||||||
}}
|
}}
|
||||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||||
|
|
||||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -495,12 +646,12 @@ func TestRunnerPrepareFileBackedPromptBodiesRenderCorrectly(t *testing.T) {
|
|||||||
llmClient := &fakeLLM{forbid: true}
|
llmClient := &fakeLLM{forbid: true}
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
promptdef.NewFilesystemRepository(promptDir),
|
promptdef.NewFilesystemRepository(promptDir),
|
||||||
profile.NewFilesystemRepository(profileDir),
|
profile.NewFilesystemRepository(profileDir), nil,
|
||||||
|
|
||||||
reader,
|
reader,
|
||||||
prompt.NewGoRenderer(),
|
prompt.NewGoRenderer(),
|
||||||
llmClient,
|
llmClient,
|
||||||
nil,
|
nil)
|
||||||
)
|
|
||||||
|
|
||||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
PromptID: "valid-file-backed",
|
PromptID: "valid-file-backed",
|
||||||
@@ -526,12 +677,13 @@ func TestRunnerPrepareRequiredInputMissingFails(t *testing.T) {
|
|||||||
|
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: def},
|
&fakePromptRepo{def: def},
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
prompt.NewGoRenderer(),
|
prompt.NewGoRenderer(),
|
||||||
&fakeLLM{forbid: true},
|
&fakeLLM{forbid: true},
|
||||||
nil,
|
nil)
|
||||||
)
|
|
||||||
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
@@ -552,12 +704,13 @@ func TestRunnerPrepareUnknownTemplateInputReferenceFails(t *testing.T) {
|
|||||||
|
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: def},
|
&fakePromptRepo{def: def},
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
prompt.NewGoRenderer(),
|
prompt.NewGoRenderer(),
|
||||||
&fakeLLM{forbid: true},
|
&fakeLLM{forbid: true},
|
||||||
nil,
|
nil)
|
||||||
)
|
|
||||||
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
@@ -579,7 +732,7 @@ func TestRunnerPrepareAPIKeyEnvNameIncludedButNotResolvedValue(t *testing.T) {
|
|||||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName},
|
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName},
|
||||||
}}
|
}}
|
||||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||||
|
|
||||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -607,12 +760,12 @@ func TestRunnerPrepareJSONSchemaBuildsStructuredOutputSpec(t *testing.T) {
|
|||||||
}
|
}
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: def},
|
&fakePromptRepo{def: def},
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
defaultRenderer(),
|
defaultRenderer(),
|
||||||
&fakeLLM{forbid: true},
|
&fakeLLM{forbid: true},
|
||||||
validator,
|
validator)
|
||||||
)
|
|
||||||
|
|
||||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -651,12 +804,12 @@ func TestRunnerPrepareJSONSchemaSchemaLoadFailureReturnsValidationError(t *testi
|
|||||||
validator := &fakeValidator{schemaErr: errors.New("schema unavailable")}
|
validator := &fakeValidator{schemaErr: errors.New("schema unavailable")}
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: def},
|
&fakePromptRepo{def: def},
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
defaultRenderer(),
|
defaultRenderer(),
|
||||||
&fakeLLM{forbid: true},
|
&fakeLLM{forbid: true},
|
||||||
validator,
|
validator)
|
||||||
)
|
|
||||||
|
|
||||||
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -681,12 +834,12 @@ func TestRunnerRunJSONSchemaSchemaLoadFailureFailsBeforeLLM(t *testing.T) {
|
|||||||
validator := &fakeValidator{schemaErr: errors.New("schema unavailable")}
|
validator := &fakeValidator{schemaErr: errors.New("schema unavailable")}
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: def},
|
&fakePromptRepo{def: def},
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
defaultRenderer(),
|
defaultRenderer(),
|
||||||
llmClient,
|
llmClient,
|
||||||
validator,
|
validator)
|
||||||
)
|
|
||||||
|
|
||||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -836,7 +989,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
|
|||||||
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
||||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}}
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}}
|
||||||
|
|
||||||
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil)
|
||||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
PromptVersion: "1",
|
PromptVersion: "1",
|
||||||
@@ -886,6 +1039,9 @@ func TestRunnerRunSuccessful(t *testing.T) {
|
|||||||
if llmClient.lastReq.Prompt.SessionID != "session-123" {
|
if llmClient.lastReq.Prompt.SessionID != "session-123" {
|
||||||
t.Fatalf("expected session id to be sent to llm, got %q", llmClient.lastReq.Prompt.SessionID)
|
t.Fatalf("expected session id to be sent to llm, got %q", llmClient.lastReq.Prompt.SessionID)
|
||||||
}
|
}
|
||||||
|
if res.SessionID != "session-123" {
|
||||||
|
t.Fatalf("expected session id in run result, got %q", res.SessionID)
|
||||||
|
}
|
||||||
if res.Usage.TotalTokens != 7 {
|
if res.Usage.TotalTokens != 7 {
|
||||||
t.Fatalf("expected token usage to be retained, got %+v", res.Usage)
|
t.Fatalf("expected token usage to be retained, got %+v", res.Usage)
|
||||||
}
|
}
|
||||||
@@ -915,7 +1071,7 @@ func TestRunnerRunPassesExtraParamsToGenerateRequestTarget(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}}
|
}}
|
||||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||||
|
|
||||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -941,7 +1097,7 @@ func TestRunnerRunAndPrepareResolveSameProfileAndEffectiveSettings(t *testing.T)
|
|||||||
}}
|
}}
|
||||||
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
||||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap"}}
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap"}}
|
||||||
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil)
|
||||||
|
|
||||||
req := domain.RunRequest{
|
req := domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -1071,7 +1227,7 @@ func TestRunnerRunExplicitRuntimeOverrideBeatsSelectedProfileValue(t *testing.T)
|
|||||||
},
|
},
|
||||||
}}
|
}}
|
||||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||||
|
|
||||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -1115,7 +1271,7 @@ func TestRunnerRunSelectedProfileBeatsBuiltInDefault(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}}
|
}}
|
||||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||||
|
|
||||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -1142,7 +1298,7 @@ func TestRunnerRunBuiltInDefaultsUsedWhenProfileOmitsOptionalFields(t *testing.T
|
|||||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"},
|
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"},
|
||||||
}}
|
}}
|
||||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||||
|
|
||||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -1172,7 +1328,7 @@ func TestRunnerRunAPIKeyEnvResolvesFromEnvironment(t *testing.T) {
|
|||||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_TEST_API_KEY"},
|
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_TEST_API_KEY"},
|
||||||
}}
|
}}
|
||||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||||
|
|
||||||
res, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
res, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1188,7 +1344,7 @@ func TestRunnerRunAPIKeyEnvMissingEnvironmentValueFailsClearly(t *testing.T) {
|
|||||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_MISSING_KEY"},
|
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_MISSING_KEY"},
|
||||||
}}
|
}}
|
||||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||||
|
|
||||||
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||||
if !errors.Is(err, ErrInvalidRequest) {
|
if !errors.Is(err, ErrInvalidRequest) {
|
||||||
@@ -1209,7 +1365,7 @@ func TestRunnerRunDirectAPIKeyBypassesMissingEnvAndReachesLLM(t *testing.T) {
|
|||||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_MISSING_KEY"},
|
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_MISSING_KEY"},
|
||||||
}}
|
}}
|
||||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||||
|
|
||||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -1233,7 +1389,7 @@ func TestRunnerPrepareAPIKeyRequiredFailsWithoutDirectKey(t *testing.T) {
|
|||||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true},
|
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true},
|
||||||
}}
|
}}
|
||||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||||
|
|
||||||
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -1252,7 +1408,7 @@ func TestRunnerRunAPIKeyRequiredSucceedsWithDirectKey(t *testing.T) {
|
|||||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true},
|
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true},
|
||||||
}}
|
}}
|
||||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||||
|
|
||||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -1279,7 +1435,7 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideWorks(t *testing.T) {
|
|||||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"},
|
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"},
|
||||||
}}
|
}}
|
||||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||||
|
|
||||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -1304,7 +1460,7 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideBeatsProfile(t *testing.T) {
|
|||||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: profileEnv},
|
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: profileEnv},
|
||||||
}}
|
}}
|
||||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||||
|
|
||||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -1328,7 +1484,7 @@ func TestRunnerRunAPIKeyValueNotPresentInMetadata(t *testing.T) {
|
|||||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName},
|
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName},
|
||||||
}}
|
}}
|
||||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||||
|
|
||||||
res, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
res, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1344,7 +1500,7 @@ func TestRunnerRunAPIKeyValueNotPresentInMetadata(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerRunPromptLoadFailure(t *testing.T) {
|
func TestRunnerRunPromptLoadFailure(t *testing.T) {
|
||||||
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
|
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
|
||||||
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"})
|
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"})
|
||||||
if !errors.Is(err, ErrPromptLoad) {
|
if !errors.Is(err, ErrPromptLoad) {
|
||||||
t.Fatalf("expected ErrPromptLoad, got %v", err)
|
t.Fatalf("expected ErrPromptLoad, got %v", err)
|
||||||
@@ -1357,12 +1513,12 @@ func TestRunnerRunPromptLoadFailure(t *testing.T) {
|
|||||||
func TestRunnerRunArtifactLoadFailure(t *testing.T) {
|
func TestRunnerRunArtifactLoadFailure(t *testing.T) {
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||||
|
|
||||||
&fakeArtifactReader{errByURI: map[string]error{"a://bad": errors.New("read failed")}},
|
&fakeArtifactReader{errByURI: map[string]error{"a://bad": errors.New("read failed")}},
|
||||||
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
|
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
|
||||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
|
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
|
||||||
nil,
|
nil)
|
||||||
)
|
|
||||||
|
|
||||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -1377,12 +1533,13 @@ func TestRunnerRunArtifactLoadFailure(t *testing.T) {
|
|||||||
func TestRunnerRunPromptRenderFailure(t *testing.T) {
|
func TestRunnerRunPromptRenderFailure(t *testing.T) {
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
&fakeRenderer{err: errors.New("render failed")},
|
&fakeRenderer{err: errors.New("render failed")},
|
||||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
|
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
|
||||||
nil,
|
nil)
|
||||||
)
|
|
||||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
@@ -1396,12 +1553,13 @@ func TestRunnerRunPromptRenderFailure(t *testing.T) {
|
|||||||
func TestRunnerRunLLMFailure(t *testing.T) {
|
func TestRunnerRunLLMFailure(t *testing.T) {
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
defaultRenderer(),
|
defaultRenderer(),
|
||||||
&fakeLLM{err: errors.New("llm failed")},
|
&fakeLLM{err: errors.New("llm failed")},
|
||||||
nil,
|
nil)
|
||||||
)
|
|
||||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
@@ -1415,12 +1573,13 @@ func TestRunnerRunLLMFailure(t *testing.T) {
|
|||||||
func TestRunnerRunCancellationPreservesGenerationCategory(t *testing.T) {
|
func TestRunnerRunCancellationPreservesGenerationCategory(t *testing.T) {
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
defaultRenderer(),
|
defaultRenderer(),
|
||||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "ignored"}},
|
&fakeLLM{resp: &domain.GenerateResponse{Content: "ignored"}},
|
||||||
nil,
|
nil)
|
||||||
)
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
cancel()
|
cancel()
|
||||||
|
|
||||||
@@ -1440,12 +1599,13 @@ func TestRunnerRunCancellationPreservesGenerationCategory(t *testing.T) {
|
|||||||
func TestRunnerRunLLMInvalidRequestMapsToUsecaseInvalidRequest(t *testing.T) {
|
func TestRunnerRunLLMInvalidRequestMapsToUsecaseInvalidRequest(t *testing.T) {
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
defaultRenderer(),
|
defaultRenderer(),
|
||||||
&fakeLLM{err: llm.ErrInvalidRequest},
|
&fakeLLM{err: llm.ErrInvalidRequest},
|
||||||
nil,
|
nil)
|
||||||
)
|
|
||||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
@@ -1463,12 +1623,13 @@ func TestRunnerRunValidationStillWorks(t *testing.T) {
|
|||||||
validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}}
|
validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}}
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
defaultRenderer(),
|
defaultRenderer(),
|
||||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}},
|
&fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}},
|
||||||
validator,
|
validator)
|
||||||
)
|
|
||||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
@@ -1489,14 +1650,17 @@ func TestRunnerRunStructuredRepairRemainsBoundedAndUsesEffectiveModelSettings(t
|
|||||||
runner := NewRunnerWithRepairer(
|
runner := NewRunnerWithRepairer(
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
|
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", TimeoutSeconds: 55},
|
"exec": {ID: "exec", BackendID: "custom", Model: "profile-model", TimeoutSeconds: 55},
|
||||||
|
}}, fakeBackendResolver{backends: map[string]domain.Backend{
|
||||||
|
"custom": {ID: "custom", Endpoint: "http://backend/v1"},
|
||||||
}},
|
}},
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
defaultRenderer(),
|
defaultRenderer(),
|
||||||
llmClient,
|
llmClient,
|
||||||
validate.NewStandardValidator("."),
|
validate.NewStandardValidator("."),
|
||||||
repairer,
|
repairer)
|
||||||
)
|
|
||||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
@@ -1518,6 +1682,47 @@ func TestRunnerRunStructuredRepairRemainsBoundedAndUsesEffectiveModelSettings(t
|
|||||||
if repairer.reqs[0].Target.TimeoutSeconds != 22 {
|
if repairer.reqs[0].Target.TimeoutSeconds != 22 {
|
||||||
t.Fatalf("expected repair to use effective timeout, got %d", repairer.reqs[0].Target.TimeoutSeconds)
|
t.Fatalf("expected repair to use effective timeout, got %d", repairer.reqs[0].Target.TimeoutSeconds)
|
||||||
}
|
}
|
||||||
|
if llmClient.lastReq.Target.BackendID != "custom" ||
|
||||||
|
repairer.reqs[0].Target.BackendID != "custom" ||
|
||||||
|
res.SelectedBackendID != "custom" {
|
||||||
|
t.Fatalf("expected backend identity in generation, repair, and result: generate=%q repair=%q result=%q",
|
||||||
|
llmClient.lastReq.Target.BackendID, repairer.reqs[0].Target.BackendID, res.SelectedBackendID)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerRunRepairCarriesEffectiveSessionID(t *testing.T) {
|
||||||
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: `{"broken":`}}
|
||||||
|
runner := NewRunnerWithRepairer(
|
||||||
|
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
|
||||||
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"exec": {ID: "exec", Endpoint: "http://example.test/v1", Model: "model"},
|
||||||
|
}},
|
||||||
|
nil,
|
||||||
|
defaultArtifactReader(),
|
||||||
|
defaultRenderer(),
|
||||||
|
llmClient,
|
||||||
|
validate.NewStandardValidator("."),
|
||||||
|
NewDefaultOutputRepairer(llmClient),
|
||||||
|
)
|
||||||
|
|
||||||
|
result, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p",
|
||||||
|
ProfileID: "exec",
|
||||||
|
SessionID: " repair-session ",
|
||||||
|
Inputs: singleInputRef(),
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected no error, got %v", err)
|
||||||
|
}
|
||||||
|
if llmClient.calls != 2 {
|
||||||
|
t.Fatalf("expected initial generation and one repair, got %d calls", llmClient.calls)
|
||||||
|
}
|
||||||
|
if llmClient.lastReq.Prompt.SessionID != "repair-session" {
|
||||||
|
t.Fatalf("expected repair generation to retain effective session, got %q", llmClient.lastReq.Prompt.SessionID)
|
||||||
|
}
|
||||||
|
if result.SessionID != "repair-session" {
|
||||||
|
t.Fatalf("expected result to retain effective session, got %q", result.SessionID)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerRunJSONSchemaRepairCarriesStructuredOutputSpec(t *testing.T) {
|
func TestRunnerRunJSONSchemaRepairCarriesStructuredOutputSpec(t *testing.T) {
|
||||||
@@ -1546,13 +1751,13 @@ func TestRunnerRunJSONSchemaRepairCarriesStructuredOutputSpec(t *testing.T) {
|
|||||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: `{"events":[1]}`}}
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: `{"events":[1]}`}}
|
||||||
runner := NewRunnerWithRepairer(
|
runner := NewRunnerWithRepairer(
|
||||||
&fakePromptRepo{def: def},
|
&fakePromptRepo{def: def},
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
defaultRenderer(),
|
defaultRenderer(),
|
||||||
llmClient,
|
llmClient,
|
||||||
validator,
|
validator,
|
||||||
repairer,
|
repairer)
|
||||||
)
|
|
||||||
|
|
||||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -1629,13 +1834,12 @@ func TestResolveExecutionTargetProfileValuesPopulateAllSupportedFields(t *testin
|
|||||||
ServiceTier: "priority",
|
ServiceTier: "priority",
|
||||||
ReasoningEffort: "low",
|
ReasoningEffort: "low",
|
||||||
APIKeyEnv: "PROFILE_KEY",
|
APIKeyEnv: "PROFILE_KEY",
|
||||||
APIKeyRequired: true,
|
|
||||||
ExtraParams: map[string]any{
|
ExtraParams: map[string]any{
|
||||||
"profile_option": "enabled",
|
"profile_option": "enabled",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
target, presence, err := resolveExecutionTarget(profileValue, nil)
|
target, presence, err := resolveExecutionTarget(nil, profileValue, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -1683,14 +1887,14 @@ func TestResolveExecutionTargetRuntimeOverridesBeatProfileForAllOverrideableFiel
|
|||||||
TopP: float64Ptr(0.5),
|
TopP: float64Ptr(0.5),
|
||||||
TimeoutSeconds: intPtr(30),
|
TimeoutSeconds: intPtr(30),
|
||||||
ServiceTier: "flex",
|
ServiceTier: "flex",
|
||||||
ReasoningEffort: "high",
|
ReasoningEffort: stringPtr("high"),
|
||||||
APIKeyEnv: "RUNTIME_KEY",
|
APIKeyEnv: "RUNTIME_KEY",
|
||||||
ExtraParams: map[string]any{
|
ExtraParams: map[string]any{
|
||||||
"runtime_only": "yes",
|
"runtime_only": "yes",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
target, presence, err := resolveExecutionTarget(profileValue, override)
|
target, presence, err := resolveExecutionTarget(nil, profileValue, override)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -1704,7 +1908,7 @@ func TestResolveExecutionTargetRuntimeOverridesBeatProfileForAllOverrideableFiel
|
|||||||
target.TopP != *override.TopP ||
|
target.TopP != *override.TopP ||
|
||||||
target.TimeoutSeconds != *override.TimeoutSeconds ||
|
target.TimeoutSeconds != *override.TimeoutSeconds ||
|
||||||
target.ServiceTier != override.ServiceTier ||
|
target.ServiceTier != override.ServiceTier ||
|
||||||
target.ReasoningEffort != override.ReasoningEffort ||
|
target.ReasoningEffort != *override.ReasoningEffort ||
|
||||||
target.APIKeyEnv != override.APIKeyEnv {
|
target.APIKeyEnv != override.APIKeyEnv {
|
||||||
t.Fatalf("expected runtime overrides to win for all fields, got %+v", target)
|
t.Fatalf("expected runtime overrides to win for all fields, got %+v", target)
|
||||||
}
|
}
|
||||||
@@ -1713,6 +1917,46 @@ func TestResolveExecutionTargetRuntimeOverridesBeatProfileForAllOverrideableFiel
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestResolveExecutionTargetReasoningOverrideStates(t *testing.T) {
|
||||||
|
profileValue := &domain.ExecutionProfile{
|
||||||
|
ReasoningEffort: "medium",
|
||||||
|
}
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
override *string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "nil inherits profile value",
|
||||||
|
want: "medium",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "nonblank replaces and trims profile value",
|
||||||
|
override: stringPtr(" high "),
|
||||||
|
want: "high",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "blank clears profile value",
|
||||||
|
override: stringPtr(" \t "),
|
||||||
|
want: "",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
target, _, err := resolveExecutionTarget(nil, profileValue, &domain.ExecutionTargetOverride{
|
||||||
|
ReasoningEffort: tt.override,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolve execution target: %v", err)
|
||||||
|
}
|
||||||
|
if target.ReasoningEffort != tt.want {
|
||||||
|
t.Fatalf("reasoning effort = %q, want %q", target.ReasoningEffort, tt.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestMergeExecutionTargetEmptyStringOverridesDoNotErase(t *testing.T) {
|
func TestMergeExecutionTargetEmptyStringOverridesDoNotErase(t *testing.T) {
|
||||||
base := domain.ExecutionTarget{
|
base := domain.ExecutionTarget{
|
||||||
Endpoint: "http://base/v1",
|
Endpoint: "http://base/v1",
|
||||||
@@ -1813,6 +2057,118 @@ func defaultExecutionProfile() *domain.ExecutionProfile {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestResolveExecutionTargetUsesBackendProfileAndRequestPrecedence(t *testing.T) {
|
||||||
|
backendValue := &domain.Backend{
|
||||||
|
ID: "custom",
|
||||||
|
Endpoint: "http://backend/v1",
|
||||||
|
APIKeyEnv: "BACKEND_KEY",
|
||||||
|
ExtraParams: map[string]any{"backend": true},
|
||||||
|
}
|
||||||
|
profileValue := &domain.ExecutionProfile{
|
||||||
|
ID: "exec",
|
||||||
|
BackendID: "custom",
|
||||||
|
Endpoint: "http://profile/v1",
|
||||||
|
Model: "profile-model",
|
||||||
|
APIKeyEnv: "PROFILE_KEY",
|
||||||
|
ExtraParams: map[string]any{"profile": true},
|
||||||
|
}
|
||||||
|
override := &domain.ExecutionTargetOverride{
|
||||||
|
Endpoint: "http://request/v1",
|
||||||
|
APIKeyEnv: "REQUEST_KEY",
|
||||||
|
ExtraParams: map[string]any{"request": true},
|
||||||
|
}
|
||||||
|
|
||||||
|
target, _, err := resolveExecutionTarget(backendValue, profileValue, override)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolve target: %v", err)
|
||||||
|
}
|
||||||
|
if target.BackendID != "custom" {
|
||||||
|
t.Fatalf("endpoint override changed backend identity: %+v", target)
|
||||||
|
}
|
||||||
|
if target.Endpoint != "http://request/v1" || target.APIKeyEnv != "REQUEST_KEY" {
|
||||||
|
t.Fatalf("request values did not win: %+v", target)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(target.ExtraParams, map[string]any{"request": true}) {
|
||||||
|
t.Fatalf("expected whole-map request replacement, got %#v", target.ExtraParams)
|
||||||
|
}
|
||||||
|
|
||||||
|
target, _, err = resolveExecutionTarget(backendValue, &domain.ExecutionProfile{
|
||||||
|
ID: "exec", BackendID: "custom", Model: "profile-model",
|
||||||
|
}, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolve backend defaults: %v", err)
|
||||||
|
}
|
||||||
|
if target.Endpoint != backendValue.Endpoint ||
|
||||||
|
target.APIKeyEnv != backendValue.APIKeyEnv ||
|
||||||
|
!reflect.DeepEqual(target.ExtraParams, backendValue.ExtraParams) {
|
||||||
|
t.Fatalf("backend defaults were not inherited: %+v", target)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerPrepareBackendResolutionAndCredentialPrecedence(t *testing.T) {
|
||||||
|
resolver := fakeBackendResolver{backends: map[string]domain.Backend{
|
||||||
|
"custom": {ID: "custom", Endpoint: "http://backend/v1", APIKeyEnv: "BACKEND_KEY"},
|
||||||
|
}}
|
||||||
|
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
||||||
|
|
||||||
|
t.Run("unknown backend is a profile load failure", func(t *testing.T) {
|
||||||
|
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"exec": {ID: "exec", BackendID: "unknown", Model: "model"},
|
||||||
|
}}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||||
|
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||||
|
if !errors.Is(err, ErrProfileLoad) {
|
||||||
|
t.Fatalf("expected ErrProfileLoad, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("nil resolver is a profile load failure", func(t *testing.T) {
|
||||||
|
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"exec": {ID: "exec", BackendID: "custom", Model: "model"},
|
||||||
|
}}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||||
|
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||||
|
if !errors.Is(err, ErrProfileLoad) {
|
||||||
|
t.Fatalf("expected ErrProfileLoad, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("request environment wins", func(t *testing.T) {
|
||||||
|
t.Setenv("REQUEST_KEY", "request-secret")
|
||||||
|
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"exec": {ID: "exec", BackendID: "custom", Model: "model", APIKeyEnv: "PROFILE_KEY"},
|
||||||
|
}}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||||
|
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(),
|
||||||
|
Execution: &domain.ExecutionTargetOverride{APIKeyEnv: "REQUEST_KEY"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare: %v", err)
|
||||||
|
}
|
||||||
|
if prepared.EffectiveModelParams.APIKeyEnv != "REQUEST_KEY" {
|
||||||
|
t.Fatalf("unexpected credential source: %+v", prepared.EffectiveModelParams)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("required direct key clears inherited environment", func(t *testing.T) {
|
||||||
|
t.Setenv("BACKEND_KEY", "backend-secret")
|
||||||
|
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"exec": {ID: "exec", BackendID: "custom", Model: "model", APIKeyRequired: true},
|
||||||
|
}}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||||
|
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||||
|
if !errors.Is(err, ErrAPIKeyRequired) {
|
||||||
|
t.Fatalf("expected ErrAPIKeyRequired, got %v", err)
|
||||||
|
}
|
||||||
|
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(), APIKey: "direct-secret",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare with direct key: %v", err)
|
||||||
|
}
|
||||||
|
if prepared.EffectiveModelParams.APIKeyEnv != "" {
|
||||||
|
t.Fatalf("expected inherited environment name to be cleared, got %q", prepared.EffectiveModelParams.APIKeyEnv)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func defaultArtifactReader() *fakeArtifactReader {
|
func defaultArtifactReader() *fakeArtifactReader {
|
||||||
return &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{
|
return &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{
|
||||||
"a://ok": {Body: []byte("x"), Hash: hashString("x")},
|
"a://ok": {Body: []byte("x"), Hash: hashString("x")},
|
||||||
@@ -1831,6 +2187,10 @@ func float64Ptr(v float64) *float64 {
|
|||||||
return &v
|
return &v
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func stringPtr(v string) *string {
|
||||||
|
return &v
|
||||||
|
}
|
||||||
|
|
||||||
func intPtr(v int) *int {
|
func intPtr(v int) *int {
|
||||||
return &v
|
return &v
|
||||||
}
|
}
|
||||||
@@ -1838,10 +2198,11 @@ func intPtr(v int) *int {
|
|||||||
func newMinimalRunner(promptRepo *fakePromptRepo, execRepo *fakeExecutionProfileRepo) *Runner {
|
func newMinimalRunner(promptRepo *fakePromptRepo, execRepo *fakeExecutionProfileRepo) *Runner {
|
||||||
return NewRunner(
|
return NewRunner(
|
||||||
promptRepo,
|
promptRepo,
|
||||||
execRepo,
|
execRepo, nil,
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
defaultRenderer(),
|
defaultRenderer(),
|
||||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
|
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
|
||||||
nil,
|
nil)
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|||||||
8
json.go
8
json.go
@@ -26,6 +26,7 @@ func (r PreparedRun) MarshalJSON() ([]byte, error) {
|
|||||||
PromptVersion string `json:"prompt_version,omitempty"`
|
PromptVersion string `json:"prompt_version,omitempty"`
|
||||||
PromptHash string `json:"prompt_hash,omitempty"`
|
PromptHash string `json:"prompt_hash,omitempty"`
|
||||||
SelectedProfileID string `json:"selected_profile_id"`
|
SelectedProfileID string `json:"selected_profile_id"`
|
||||||
|
SelectedBackendID string `json:"selected_backend_id,omitempty"`
|
||||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||||
OutputContract OutputContract `json:"output_contract"`
|
OutputContract OutputContract `json:"output_contract"`
|
||||||
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
||||||
@@ -41,6 +42,7 @@ func (r PreparedRun) MarshalJSON() ([]byte, error) {
|
|||||||
PromptVersion: r.PromptVersion,
|
PromptVersion: r.PromptVersion,
|
||||||
PromptHash: r.PromptHash,
|
PromptHash: r.PromptHash,
|
||||||
SelectedProfileID: r.SelectedProfileID,
|
SelectedProfileID: r.SelectedProfileID,
|
||||||
|
SelectedBackendID: r.SelectedBackendID,
|
||||||
EffectiveModelParams: r.EffectiveModelParams,
|
EffectiveModelParams: r.EffectiveModelParams,
|
||||||
OutputContract: r.OutputContract,
|
OutputContract: r.OutputContract,
|
||||||
StructuredOutput: r.StructuredOutput,
|
StructuredOutput: r.StructuredOutput,
|
||||||
@@ -79,8 +81,10 @@ func (r RunResult) MarshalJSON() ([]byte, error) {
|
|||||||
PromptID: r.PromptID,
|
PromptID: r.PromptID,
|
||||||
PromptVersion: r.PromptVersion,
|
PromptVersion: r.PromptVersion,
|
||||||
PromptHash: r.PromptHash,
|
PromptHash: r.PromptHash,
|
||||||
|
SessionID: r.SessionID,
|
||||||
RenderedPromptHash: r.RenderedPromptHash,
|
RenderedPromptHash: r.RenderedPromptHash,
|
||||||
SelectedProfileID: r.SelectedProfileID,
|
SelectedProfileID: r.SelectedProfileID,
|
||||||
|
SelectedBackendID: r.SelectedBackendID,
|
||||||
ModelName: r.ModelName,
|
ModelName: r.ModelName,
|
||||||
Endpoint: r.Endpoint,
|
Endpoint: r.Endpoint,
|
||||||
EffectiveModelParams: r.EffectiveModelParams,
|
EffectiveModelParams: r.EffectiveModelParams,
|
||||||
@@ -108,8 +112,10 @@ func (r *RunResult) UnmarshalJSON(data []byte) error {
|
|||||||
PromptID: wire.PromptID,
|
PromptID: wire.PromptID,
|
||||||
PromptVersion: wire.PromptVersion,
|
PromptVersion: wire.PromptVersion,
|
||||||
PromptHash: wire.PromptHash,
|
PromptHash: wire.PromptHash,
|
||||||
|
SessionID: wire.SessionID,
|
||||||
RenderedPromptHash: wire.RenderedPromptHash,
|
RenderedPromptHash: wire.RenderedPromptHash,
|
||||||
SelectedProfileID: wire.SelectedProfileID,
|
SelectedProfileID: wire.SelectedProfileID,
|
||||||
|
SelectedBackendID: wire.SelectedBackendID,
|
||||||
ModelName: wire.ModelName,
|
ModelName: wire.ModelName,
|
||||||
Endpoint: wire.Endpoint,
|
Endpoint: wire.Endpoint,
|
||||||
EffectiveModelParams: wire.EffectiveModelParams,
|
EffectiveModelParams: wire.EffectiveModelParams,
|
||||||
@@ -134,8 +140,10 @@ type runResultJSON struct {
|
|||||||
PromptID string `json:"prompt_id"`
|
PromptID string `json:"prompt_id"`
|
||||||
PromptVersion string `json:"prompt_version,omitempty"`
|
PromptVersion string `json:"prompt_version,omitempty"`
|
||||||
PromptHash string `json:"prompt_hash,omitempty"`
|
PromptHash string `json:"prompt_hash,omitempty"`
|
||||||
|
SessionID string `json:"session_id,omitempty"`
|
||||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||||
SelectedProfileID string `json:"selected_profile_id"`
|
SelectedProfileID string `json:"selected_profile_id"`
|
||||||
|
SelectedBackendID string `json:"selected_backend_id,omitempty"`
|
||||||
ModelName string `json:"model_name"`
|
ModelName string `json:"model_name"`
|
||||||
Endpoint string `json:"endpoint"`
|
Endpoint string `json:"endpoint"`
|
||||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||||
|
|||||||
12
profiles.go
12
profiles.go
@@ -7,6 +7,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -15,7 +16,8 @@ import (
|
|||||||
//
|
//
|
||||||
// It does not register global state, maintain a model catalog, or resolve
|
// It does not register global state, maintain a model catalog, or resolve
|
||||||
// credentials. If APIKeyRequired is true, callers satisfy it with
|
// credentials. If APIKeyRequired is true, callers satisfy it with
|
||||||
// RunRequest.APIKey. Raw API keys do not belong in profiles.
|
// RunRequest.APIKey or an explicit request ExecutionTargetOverride.APIKeyEnv.
|
||||||
|
// Raw API keys do not belong in profiles.
|
||||||
//
|
//
|
||||||
// The function copies the ExtraParams map itself but does not recursively copy
|
// The function copies the ExtraParams map itself but does not recursively copy
|
||||||
// nested values. Validation and a deep copy occur when NewEngine applies a
|
// nested values. Validation and a deep copy occur when NewEngine applies a
|
||||||
@@ -23,6 +25,7 @@ import (
|
|||||||
func OpenAICompatibleProfile(cfg OpenAICompatibleProfileConfig) Profile {
|
func OpenAICompatibleProfile(cfg OpenAICompatibleProfileConfig) Profile {
|
||||||
return Profile{
|
return Profile{
|
||||||
ID: cfg.ID,
|
ID: cfg.ID,
|
||||||
|
BackendID: cfg.BackendID,
|
||||||
Endpoint: cfg.Endpoint,
|
Endpoint: cfg.Endpoint,
|
||||||
Model: cfg.Model,
|
Model: cfg.Model,
|
||||||
Temperature: cfg.Temperature,
|
Temperature: cfg.Temperature,
|
||||||
@@ -79,12 +82,13 @@ func (r *memoryProfileRepository) GetProfile(_ context.Context, id string) (*dom
|
|||||||
}
|
}
|
||||||
|
|
||||||
func toDomainProfile(publicProfile Profile) (domain.ExecutionProfile, error) {
|
func toDomainProfile(publicProfile Profile) (domain.ExecutionProfile, error) {
|
||||||
extraParams, err := copyPublicJSONMap(publicProfile.ExtraParams)
|
extraParams, err := jsonvalue.CopyMap(publicProfile.ExtraParams)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return domain.ExecutionProfile{}, err
|
return domain.ExecutionProfile{}, err
|
||||||
}
|
}
|
||||||
prof := domain.ExecutionProfile{
|
prof := domain.ExecutionProfile{
|
||||||
ID: strings.TrimSpace(publicProfile.ID),
|
ID: strings.TrimSpace(publicProfile.ID),
|
||||||
|
BackendID: strings.TrimSpace(publicProfile.BackendID),
|
||||||
Endpoint: publicProfile.Endpoint,
|
Endpoint: publicProfile.Endpoint,
|
||||||
Model: publicProfile.Model,
|
Model: publicProfile.Model,
|
||||||
Temperature: publicProfile.Temperature,
|
Temperature: publicProfile.Temperature,
|
||||||
@@ -106,8 +110,8 @@ func validatePublicProfile(prof domain.ExecutionProfile) error {
|
|||||||
if strings.TrimSpace(prof.ID) == "" {
|
if strings.TrimSpace(prof.ID) == "" {
|
||||||
return errors.New("id is required")
|
return errors.New("id is required")
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(prof.Endpoint) == "" {
|
if strings.TrimSpace(prof.BackendID) == "" && strings.TrimSpace(prof.Endpoint) == "" {
|
||||||
return errors.New("endpoint is required")
|
return errors.New("backend or endpoint is required")
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(prof.Model) == "" {
|
if strings.TrimSpace(prof.Model) == "" {
|
||||||
return errors.New("model is required")
|
return errors.New("model is required")
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package promptkit_test
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -26,6 +27,356 @@ func TestPreparedRunJSONOmitsZeroTimingValues(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestBackendIdentityJSONNamesAndOmission(t *testing.T) {
|
||||||
|
t.Run("execution target round trip", func(t *testing.T) {
|
||||||
|
value := promptkit.ExecutionTarget{BackendID: promptkit.BackendOpenRouter}
|
||||||
|
payload, err := json.Marshal(value)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal execution target: %v", err)
|
||||||
|
}
|
||||||
|
var decoded promptkit.ExecutionTarget
|
||||||
|
if err := json.Unmarshal(payload, &decoded); err != nil {
|
||||||
|
t.Fatalf("unmarshal execution target: %v", err)
|
||||||
|
}
|
||||||
|
if decoded.BackendID != value.BackendID {
|
||||||
|
t.Fatalf("backend identity did not round trip: got %q want %q", decoded.BackendID, value.BackendID)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("prepared run round trip", func(t *testing.T) {
|
||||||
|
value := promptkit.PreparedRun{SelectedBackendID: promptkit.BackendOpenRouter}
|
||||||
|
payload, err := json.Marshal(value)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal prepared run: %v", err)
|
||||||
|
}
|
||||||
|
var decoded promptkit.PreparedRun
|
||||||
|
if err := json.Unmarshal(payload, &decoded); err != nil {
|
||||||
|
t.Fatalf("unmarshal prepared run: %v", err)
|
||||||
|
}
|
||||||
|
if decoded.SelectedBackendID != value.SelectedBackendID {
|
||||||
|
t.Fatalf("backend identity did not round trip: got %q want %q", decoded.SelectedBackendID, value.SelectedBackendID)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("run result round trip", func(t *testing.T) {
|
||||||
|
value := promptkit.RunResult{SelectedBackendID: promptkit.BackendOpenRouter}
|
||||||
|
payload, err := json.Marshal(value)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal run result: %v", err)
|
||||||
|
}
|
||||||
|
var decoded promptkit.RunResult
|
||||||
|
if err := json.Unmarshal(payload, &decoded); err != nil {
|
||||||
|
t.Fatalf("unmarshal run result: %v", err)
|
||||||
|
}
|
||||||
|
if decoded.SelectedBackendID != value.SelectedBackendID {
|
||||||
|
t.Fatalf("backend identity did not round trip: got %q want %q", decoded.SelectedBackendID, value.SelectedBackendID)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
payload, err := json.Marshal(promptkit.ExecutionTarget{})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal empty execution target: %v", err)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(payload), `"backend_id"`) {
|
||||||
|
t.Fatalf("empty backend identity was not omitted: %s", payload)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestEndpointOnlyProfileOmitsBackendIdentityFromStableJSON(t *testing.T) {
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
|
ID: "profile", Endpoint: "http://example.test/v1", Model: "model",
|
||||||
|
}),
|
||||||
|
promptkit.WithLLMClient(client),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare endpoint-only profile: %v", err)
|
||||||
|
}
|
||||||
|
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("run endpoint-only profile: %v", err)
|
||||||
|
}
|
||||||
|
if prepared.SelectedBackendID != "" ||
|
||||||
|
prepared.EffectiveModelParams.BackendID != "" ||
|
||||||
|
result.SelectedBackendID != "" ||
|
||||||
|
result.EffectiveModelParams.BackendID != "" {
|
||||||
|
t.Fatalf("endpoint-only profile acquired backend identity: prepared=%+v result=%+v", prepared, result)
|
||||||
|
}
|
||||||
|
for _, value := range []any{prepared, result} {
|
||||||
|
payload, err := json.Marshal(value)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal endpoint-only value: %v", err)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(payload), `"backend_id"`) || strings.Contains(string(payload), `"selected_backend_id"`) {
|
||||||
|
t.Fatalf("endpoint-only backend identity was not omitted: %s", payload)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUnknownProfileBackendHasProfileLoadIdentity(t *testing.T) {
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{ID: "profile", BackendID: "unknown", Model: "model"}),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||||
|
if !errors.Is(err, promptkit.ErrProfileLoad) {
|
||||||
|
t.Fatalf("expected ErrProfileLoad, got %v", err)
|
||||||
|
}
|
||||||
|
if errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("unknown backend should not have invalid-request identity: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCustomBackendFlowsThroughProfilesOverridesAndInjectedClient(t *testing.T) {
|
||||||
|
t.Setenv("CUSTOM_LLM_KEY", "test-key")
|
||||||
|
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prompt", "backend-profile", "message"), "."),
|
||||||
|
promptkit.WithBackend(promptkit.Backend{
|
||||||
|
ID: " custom ",
|
||||||
|
Endpoint: " http://backend.example/v1 ",
|
||||||
|
APIKeyEnv: " CUSTOM_LLM_KEY ",
|
||||||
|
ExtraParams: map[string]any{
|
||||||
|
"provider": "custom",
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
promptkit.WithProfiles(
|
||||||
|
promptkit.Profile{ID: "backend-profile", BackendID: "custom", Model: "backend-model"},
|
||||||
|
promptkit.Profile{ID: "profile-endpoint", BackendID: "custom", Endpoint: "http://profile.example/v1", Model: "profile-model"},
|
||||||
|
promptkit.Profile{ID: "blank-profile-endpoint", BackendID: "custom", Endpoint: " \t ", Model: "profile-model"},
|
||||||
|
),
|
||||||
|
promptkit.WithLLMClient(client),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("run with custom backend: %v", err)
|
||||||
|
}
|
||||||
|
if len(client.requests) != 1 {
|
||||||
|
t.Fatalf("expected one injected-client request, got %d", len(client.requests))
|
||||||
|
}
|
||||||
|
target := client.requests[0].Target
|
||||||
|
if target.BackendID != "custom" ||
|
||||||
|
target.Endpoint != "http://backend.example/v1" ||
|
||||||
|
target.APIKeyEnv != "CUSTOM_LLM_KEY" ||
|
||||||
|
target.Model != "backend-model" ||
|
||||||
|
target.ExtraParams["provider"] != "custom" ||
|
||||||
|
result.SelectedBackendID != "custom" {
|
||||||
|
t.Fatalf("unexpected custom backend settings: target=%+v result_backend=%q", target, result.SelectedBackendID)
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||||
|
PromptID: "prompt", ProfileID: "profile-endpoint",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare profile endpoint override: %v", err)
|
||||||
|
}
|
||||||
|
if prepared.SelectedBackendID != "custom" || prepared.EffectiveModelParams.Endpoint != "http://profile.example/v1" {
|
||||||
|
t.Fatalf("profile endpoint override changed backend identity: %+v", prepared)
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared, err = engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||||
|
PromptID: "prompt", ProfileID: "blank-profile-endpoint",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare blank profile endpoint: %v", err)
|
||||||
|
}
|
||||||
|
if prepared.SelectedBackendID != "custom" || prepared.EffectiveModelParams.Endpoint != "http://backend.example/v1" {
|
||||||
|
t.Fatalf("blank profile endpoint did not inherit backend endpoint: %+v", prepared)
|
||||||
|
}
|
||||||
|
|
||||||
|
prepared, err = engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||||
|
PromptID: "prompt",
|
||||||
|
Execution: &promptkit.ExecutionTargetOverride{
|
||||||
|
Endpoint: "http://request.example/v1",
|
||||||
|
},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare request endpoint override: %v", err)
|
||||||
|
}
|
||||||
|
if prepared.SelectedBackendID != "custom" || prepared.EffectiveModelParams.Endpoint != "http://request.example/v1" {
|
||||||
|
t.Fatalf("request endpoint override changed backend identity: %+v", prepared)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCustomBackendSupportsFileProfileAndBothSelectionPaths(t *testing.T) {
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prompt", "file-profile", "message"), "."),
|
||||||
|
promptkit.WithProfileFS(fstest.MapFS{
|
||||||
|
"profile.yaml": &fstest.MapFile{Data: []byte(`id: file-profile
|
||||||
|
backend: file-backend
|
||||||
|
endpoint: " "
|
||||||
|
model: file-model
|
||||||
|
`)},
|
||||||
|
}, "."),
|
||||||
|
promptkit.WithBackend(promptkit.Backend{
|
||||||
|
ID: "file-backend",
|
||||||
|
Endpoint: "http://file-backend.example/v1",
|
||||||
|
}),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, request := range []promptkit.RunRequest{
|
||||||
|
{PromptID: "prompt"},
|
||||||
|
{PromptID: "prompt", ProfileID: "file-profile"},
|
||||||
|
} {
|
||||||
|
prepared, err := engine.Prepare(context.Background(), request)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare file profile: %v", err)
|
||||||
|
}
|
||||||
|
if prepared.SelectedBackendID != "file-backend" ||
|
||||||
|
prepared.EffectiveModelParams.Endpoint != "http://file-backend.example/v1" {
|
||||||
|
t.Fatalf("unexpected file-profile backend resolution: %+v", prepared)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackendOptionsAccumulateAndRegistrationsAreEngineLocal(t *testing.T) {
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prompt", "first-profile", "message"), "."),
|
||||||
|
promptkit.WithBackend(promptkit.Backend{ID: "first", Endpoint: "http://first.example/v1"}),
|
||||||
|
promptkit.WithBackend(promptkit.Backend{ID: "second", Endpoint: "http://second.example/v1"}),
|
||||||
|
promptkit.WithProfiles(
|
||||||
|
promptkit.Profile{ID: "first-profile", BackendID: "first", Model: "model"},
|
||||||
|
promptkit.Profile{ID: "second-profile", BackendID: "second", Model: "model"},
|
||||||
|
),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine with accumulated registrations: %v", err)
|
||||||
|
}
|
||||||
|
for profileID, wantEndpoint := range map[string]string{
|
||||||
|
"first-profile": "http://first.example/v1",
|
||||||
|
"second-profile": "http://second.example/v1",
|
||||||
|
} {
|
||||||
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||||
|
PromptID: "prompt", ProfileID: profileID,
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare %s: %v", profileID, err)
|
||||||
|
}
|
||||||
|
if prepared.EffectiveModelParams.Endpoint != wantEndpoint {
|
||||||
|
t.Fatalf("profile %s endpoint=%q, want %q", profileID, prepared.EffectiveModelParams.Endpoint, wantEndpoint)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
newEngine := func(endpoint string) *promptkit.Engine {
|
||||||
|
t.Helper()
|
||||||
|
value, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
|
||||||
|
promptkit.WithBackend(promptkit.Backend{ID: "same-id", Endpoint: endpoint}),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{ID: "profile", BackendID: "same-id", Model: "model"}),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct isolated engine: %v", err)
|
||||||
|
}
|
||||||
|
return value
|
||||||
|
}
|
||||||
|
firstEngine := newEngine("http://one.example/v1")
|
||||||
|
secondEngine := newEngine("http://two.example/v1")
|
||||||
|
for engine, wantEndpoint := range map[*promptkit.Engine]string{
|
||||||
|
firstEngine: "http://one.example/v1",
|
||||||
|
secondEngine: "http://two.example/v1",
|
||||||
|
} {
|
||||||
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare isolated engine: %v", err)
|
||||||
|
}
|
||||||
|
if prepared.EffectiveModelParams.Endpoint != wantEndpoint {
|
||||||
|
t.Fatalf("isolated engine endpoint=%q, want %q", prepared.EffectiveModelParams.Endpoint, wantEndpoint)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackendRegistrationRejectsInvalidAndDuplicateDefinitions(t *testing.T) {
|
||||||
|
cycle := map[string]any{}
|
||||||
|
cycle["self"] = cycle
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
backends []promptkit.Backend
|
||||||
|
}{
|
||||||
|
{name: "blank id", backends: []promptkit.Backend{{Endpoint: "http://example.test/v1"}}},
|
||||||
|
{name: "invalid endpoint", backends: []promptkit.Backend{{ID: "custom", Endpoint: "ftp://example.test/v1"}}},
|
||||||
|
{name: "invalid environment", backends: []promptkit.Backend{{ID: "custom", Endpoint: "http://example.test/v1", APIKeyEnv: "BAD-NAME"}}},
|
||||||
|
{name: "reserved extra parameter", backends: []promptkit.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: map[string]any{"model": "override"}}}},
|
||||||
|
{name: "cyclic extra parameter", backends: []promptkit.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: cycle}}},
|
||||||
|
{name: "malformed JSON number", backends: []promptkit.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: map[string]any{"value": json.Number("01")}}}},
|
||||||
|
{name: "duplicate consumer id", backends: []promptkit.Backend{
|
||||||
|
{ID: " custom ", Endpoint: "http://one.example/v1"},
|
||||||
|
{ID: "custom", Endpoint: "http://two.example/v1"},
|
||||||
|
}},
|
||||||
|
{name: "reserved built-in id", backends: []promptkit.Backend{{
|
||||||
|
ID: promptkit.BackendOpenRouter, Endpoint: "http://replacement.example/v1",
|
||||||
|
}}},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
options := []promptkit.Option{
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
|
||||||
|
}
|
||||||
|
for _, backend := range tt.backends {
|
||||||
|
options = append(options, promptkit.WithBackend(backend))
|
||||||
|
}
|
||||||
|
_, err := promptkit.NewEngine(promptkit.Config{}, options...)
|
||||||
|
if !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||||
|
t.Fatalf("expected ErrInvalidConfig, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBackendExtraParamsAreDeeplyCopiedAtConstructionAndLookup(t *testing.T) {
|
||||||
|
nested := map[string]any{"value": "original"}
|
||||||
|
extraParams := map[string]any{"nested": nested}
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
|
||||||
|
promptkit.WithBackend(promptkit.Backend{
|
||||||
|
ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: extraParams,
|
||||||
|
}),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{ID: "profile", BackendID: "custom", Model: "model"}),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine: %v", err)
|
||||||
|
}
|
||||||
|
nested["value"] = "mutated input"
|
||||||
|
extraParams["later"] = true
|
||||||
|
|
||||||
|
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("first prepare: %v", err)
|
||||||
|
}
|
||||||
|
gotNested := prepared.EffectiveModelParams.ExtraParams["nested"].(map[string]any)
|
||||||
|
if gotNested["value"] != "original" || prepared.EffectiveModelParams.ExtraParams["later"] != nil {
|
||||||
|
t.Fatalf("backend retained caller mutations: %#v", prepared.EffectiveModelParams.ExtraParams)
|
||||||
|
}
|
||||||
|
gotNested["value"] = "mutated lookup"
|
||||||
|
|
||||||
|
prepared, err = engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("second prepare: %v", err)
|
||||||
|
}
|
||||||
|
gotNested = prepared.EffectiveModelParams.ExtraParams["nested"].(map[string]any)
|
||||||
|
if gotNested["value"] != "original" {
|
||||||
|
t.Fatalf("backend retained lookup mutation: %#v", prepared.EffectiveModelParams.ExtraParams)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestPreparedRunJSONTimingRoundTrips(t *testing.T) {
|
func TestPreparedRunJSONTimingRoundTrips(t *testing.T) {
|
||||||
start := time.Date(2026, time.July, 29, 12, 0, 0, 0, time.UTC)
|
start := time.Date(2026, time.July, 29, 12, 0, 0, 0, time.UTC)
|
||||||
prepared := promptkit.PreparedRun{
|
prepared := promptkit.PreparedRun{
|
||||||
@@ -55,6 +406,7 @@ func TestRunResultJSONUsesMillisecondsAndRoundTrips(t *testing.T) {
|
|||||||
result := promptkit.RunResult{
|
result := promptkit.RunResult{
|
||||||
RunID: "opaque-run-id",
|
RunID: "opaque-run-id",
|
||||||
Artifact: promptkit.Artifact{Name: "output", ContentType: "text/plain", Body: []byte("ok")},
|
Artifact: promptkit.Artifact{Name: "output", ContentType: "text/plain", Body: []byte("ok")},
|
||||||
|
SessionID: "session-123",
|
||||||
StartTime: start,
|
StartTime: start,
|
||||||
EndTime: start.Add(1500 * time.Millisecond),
|
EndTime: start.Add(1500 * time.Millisecond),
|
||||||
Duration: 1500 * time.Millisecond,
|
Duration: 1500 * time.Millisecond,
|
||||||
@@ -74,6 +426,9 @@ func TestRunResultJSONUsesMillisecondsAndRoundTrips(t *testing.T) {
|
|||||||
if _, exists := object["duration"]; exists {
|
if _, exists := object["duration"]; exists {
|
||||||
t.Fatalf("unexpected nanosecond duration field in %s", payload)
|
t.Fatalf("unexpected nanosecond duration field in %s", payload)
|
||||||
}
|
}
|
||||||
|
if got := object["session_id"]; got != result.SessionID {
|
||||||
|
t.Fatalf("expected session_id=%q, got %#v in %s", result.SessionID, got, payload)
|
||||||
|
}
|
||||||
artifact, ok := object["artifact"].(map[string]any)
|
artifact, ok := object["artifact"].(map[string]any)
|
||||||
if !ok || artifact["content_type"] != "text/plain" {
|
if !ok || artifact["content_type"] != "text/plain" {
|
||||||
t.Fatalf("expected stable artifact JSON fields, got %#v", object["artifact"])
|
t.Fatalf("expected stable artifact JSON fields, got %#v", object["artifact"])
|
||||||
@@ -83,7 +438,10 @@ func TestRunResultJSONUsesMillisecondsAndRoundTrips(t *testing.T) {
|
|||||||
if err := json.Unmarshal(payload, &decoded); err != nil {
|
if err := json.Unmarshal(payload, &decoded); err != nil {
|
||||||
t.Fatalf("unmarshal run result: %v", err)
|
t.Fatalf("unmarshal run result: %v", err)
|
||||||
}
|
}
|
||||||
if decoded.Duration != result.Duration || !decoded.StartTime.Equal(result.StartTime) || !decoded.EndTime.Equal(result.EndTime) {
|
if decoded.SessionID != result.SessionID ||
|
||||||
|
decoded.Duration != result.Duration ||
|
||||||
|
!decoded.StartTime.Equal(result.StartTime) ||
|
||||||
|
!decoded.EndTime.Equal(result.EndTime) {
|
||||||
t.Fatalf("timing values did not round trip: got %#v, want %#v", decoded, result)
|
t.Fatalf("timing values did not round trip: got %#v, want %#v", decoded, result)
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -91,11 +449,18 @@ func TestRunResultJSONUsesMillisecondsAndRoundTrips(t *testing.T) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("marshal zero run result: %v", err)
|
t.Fatalf("marshal zero run result: %v", err)
|
||||||
}
|
}
|
||||||
for _, field := range []string{"start_time", "end_time", "duration_ms"} {
|
for _, field := range []string{"session_id", "start_time", "end_time", "duration_ms"} {
|
||||||
if strings.Contains(string(payload), `"`+field+`"`) {
|
if strings.Contains(string(payload), `"`+field+`"`) {
|
||||||
t.Fatalf("expected zero %s to be omitted, got %s", field, payload)
|
t.Fatalf("expected zero %s to be omitted, got %s", field, payload)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
var decodedEmpty promptkit.RunResult
|
||||||
|
if err := json.Unmarshal(payload, &decodedEmpty); err != nil {
|
||||||
|
t.Fatalf("unmarshal run result without session_id: %v", err)
|
||||||
|
}
|
||||||
|
if decodedEmpty.SessionID != "" {
|
||||||
|
t.Fatalf("expected absent session_id to decode empty, got %q", decodedEmpty.SessionID)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestEngineValidationIsSinglePass(t *testing.T) {
|
func TestEngineValidationIsSinglePass(t *testing.T) {
|
||||||
@@ -254,9 +619,13 @@ func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) {
|
|||||||
func TestEngineSupportsConcurrentPrepareAndRun(t *testing.T) {
|
func TestEngineSupportsConcurrentPrepareAndRun(t *testing.T) {
|
||||||
engine, err := promptkit.NewEngine(promptkit.Config{},
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
|
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
|
||||||
|
promptkit.WithBackend(promptkit.Backend{
|
||||||
|
ID: "concurrent",
|
||||||
|
Endpoint: "http://example.test/v1",
|
||||||
|
}),
|
||||||
promptkit.WithProfiles(promptkit.Profile{
|
promptkit.WithProfiles(promptkit.Profile{
|
||||||
ID: "profile",
|
ID: "profile",
|
||||||
Endpoint: "http://example.test/v1",
|
BackendID: "concurrent",
|
||||||
Model: "model",
|
Model: "model",
|
||||||
}),
|
}),
|
||||||
promptkit.WithLLMClient(countingLLMClient{}),
|
promptkit.WithLLMClient(countingLLMClient{}),
|
||||||
|
|||||||
104
types.go
104
types.go
@@ -91,6 +91,16 @@ type RunRequest struct {
|
|||||||
// profile is used; if both are empty, the error matches ErrProfileRequired
|
// profile is used; if both are empty, the error matches ErrProfileRequired
|
||||||
// and ErrInvalidRequest.
|
// and ErrInvalidRequest.
|
||||||
ProfileID string
|
ProfileID string
|
||||||
|
// SessionID optionally supplies a direct per-run session identifier. A
|
||||||
|
// nonblank value is trimmed and overrides the prompt definition's
|
||||||
|
// session_id template. A blank value supplies no direct override. The
|
||||||
|
// maximum is 256 Unicode code points after trimming. A direct value is
|
||||||
|
// opaque consumer metadata, not a credential, and may be exposed in
|
||||||
|
// prepared values, results, collaborator requests, provider requests, and
|
||||||
|
// provider observability. Callers should use stable, non-sensitive
|
||||||
|
// identifiers. An overlong direct value makes Prepare or Run return an
|
||||||
|
// error matching ErrInvalidRequest.
|
||||||
|
SessionID string
|
||||||
// APIKey is a request-scoped direct credential. It takes precedence over
|
// APIKey is a request-scoped direct credential. It takes precedence over
|
||||||
// APIKeyEnv, is passed to the selected LLMClient, and is never included in
|
// APIKeyEnv, is passed to the selected LLMClient, and is never included in
|
||||||
// prepared values, results, hashes, JSON, String, or GoString output.
|
// prepared values, results, hashes, JSON, String, or GoString output.
|
||||||
@@ -101,8 +111,8 @@ type RunRequest struct {
|
|||||||
// Vars supplies Go-template data for messages and the session ID. Nil and
|
// Vars supplies Go-template data for messages and the session ID. Nil and
|
||||||
// empty maps are equivalent.
|
// empty maps are equivalent.
|
||||||
Vars map[string]string
|
Vars map[string]string
|
||||||
// Execution optionally overrides individual profile execution settings.
|
// Execution optionally overrides individual execution settings. Nil uses
|
||||||
// Nil uses the selected profile over framework defaults.
|
// the selected profile over its backend, when any, and framework defaults.
|
||||||
Execution *ExecutionTargetOverride
|
Execution *ExecutionTargetOverride
|
||||||
// Validation optionally replaces the prompt's complete output contract. It
|
// Validation optionally replaces the prompt's complete output contract. It
|
||||||
// does not merge individual fields. Nil uses the prompt contract.
|
// does not merge individual fields. Nil uses the prompt contract.
|
||||||
@@ -126,8 +136,12 @@ type PreparedRun struct {
|
|||||||
// SelectedProfileID is the explicit request profile or prompt default that
|
// SelectedProfileID is the explicit request profile or prompt default that
|
||||||
// supplied execution settings.
|
// supplied execution settings.
|
||||||
SelectedProfileID string `json:"selected_profile_id"`
|
SelectedProfileID string `json:"selected_profile_id"`
|
||||||
|
// SelectedBackendID equals EffectiveModelParams.BackendID. It is empty for
|
||||||
|
// an endpoint-only profile.
|
||||||
|
SelectedBackendID string `json:"selected_backend_id,omitempty"`
|
||||||
// EffectiveModelParams contains framework defaults overlaid by the selected
|
// EffectiveModelParams contains framework defaults overlaid by the selected
|
||||||
// profile and then request overrides. It excludes resolved API-key values.
|
// backend, profile, and then request overrides. It excludes resolved API-key
|
||||||
|
// values.
|
||||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||||
// OutputContract is the complete effective output contract.
|
// OutputContract is the complete effective output contract.
|
||||||
OutputContract OutputContract `json:"output_contract"`
|
OutputContract OutputContract `json:"output_contract"`
|
||||||
@@ -136,7 +150,7 @@ type PreparedRun struct {
|
|||||||
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
||||||
// InputHashes maps every supplied input name to its opaque artifact hash.
|
// InputHashes maps every supplied input name to its opaque artifact hash.
|
||||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||||
// SessionID is the trimmed rendered session identifier, if any.
|
// SessionID is the effective direct or rendered session identifier, if any.
|
||||||
SessionID string `json:"session_id,omitempty"`
|
SessionID string `json:"session_id,omitempty"`
|
||||||
// RenderedPromptHash is an opaque equality value for SessionID and Messages.
|
// RenderedPromptHash is an opaque equality value for SessionID and Messages.
|
||||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||||
@@ -175,11 +189,17 @@ type RunResult struct {
|
|||||||
// PromptHash is the same opaque definition equality value exposed by
|
// PromptHash is the same opaque definition equality value exposed by
|
||||||
// PreparedRun.
|
// PreparedRun.
|
||||||
PromptHash string `json:"prompt_hash,omitempty"`
|
PromptHash string `json:"prompt_hash,omitempty"`
|
||||||
|
// SessionID is the effective direct or rendered session identifier, if any.
|
||||||
|
// JSON omits an empty value.
|
||||||
|
SessionID string `json:"session_id,omitempty"`
|
||||||
// RenderedPromptHash is the same opaque rendered-prompt equality value
|
// RenderedPromptHash is the same opaque rendered-prompt equality value
|
||||||
// computed during preparation.
|
// computed during preparation.
|
||||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||||
// SelectedProfileID identifies the profile used for execution.
|
// SelectedProfileID identifies the profile used for execution.
|
||||||
SelectedProfileID string `json:"selected_profile_id"`
|
SelectedProfileID string `json:"selected_profile_id"`
|
||||||
|
// SelectedBackendID equals EffectiveModelParams.BackendID. It is empty for
|
||||||
|
// an endpoint-only profile.
|
||||||
|
SelectedBackendID string `json:"selected_backend_id,omitempty"`
|
||||||
// ModelName is the effective model name and equals
|
// ModelName is the effective model name and equals
|
||||||
// EffectiveModelParams.Model.
|
// EffectiveModelParams.Model.
|
||||||
ModelName string `json:"model_name"`
|
ModelName string `json:"model_name"`
|
||||||
@@ -259,6 +279,11 @@ type ArtifactReader interface {
|
|||||||
// ExecutionTarget represents effective model runtime settings and has a stable
|
// ExecutionTarget represents effective model runtime settings and has a stable
|
||||||
// JSON representation. It never exposes a resolved API-key value.
|
// JSON representation. It never exposes a resolved API-key value.
|
||||||
type ExecutionTarget struct {
|
type ExecutionTarget struct {
|
||||||
|
// BackendID is the effective routing identity selected by the profile. It
|
||||||
|
// remains unchanged when a profile or request overrides Endpoint and is
|
||||||
|
// empty for endpoint-only profiles. It is supplied to injected LLMClient
|
||||||
|
// implementations as part of the effective target.
|
||||||
|
BackendID string `json:"backend_id,omitempty"`
|
||||||
// Endpoint is the model-provider base URL.
|
// Endpoint is the model-provider base URL.
|
||||||
Endpoint string `json:"endpoint"`
|
Endpoint string `json:"endpoint"`
|
||||||
// Model is the provider model identifier.
|
// Model is the provider model identifier.
|
||||||
@@ -276,7 +301,8 @@ type ExecutionTarget struct {
|
|||||||
TimeoutSeconds int `json:"timeout_seconds"`
|
TimeoutSeconds int `json:"timeout_seconds"`
|
||||||
// ServiceTier is an optional provider-specific request tier.
|
// ServiceTier is an optional provider-specific request tier.
|
||||||
ServiceTier string `json:"service_tier"`
|
ServiceTier string `json:"service_tier"`
|
||||||
// ReasoningEffort is an optional provider-specific reasoning setting.
|
// ReasoningEffort is the effective opaque provider-specific reasoning
|
||||||
|
// setting. An empty value instructs model clients to omit reasoning.
|
||||||
ReasoningEffort string `json:"reasoning_effort"`
|
ReasoningEffort string `json:"reasoning_effort"`
|
||||||
// APIKeyEnv is an environment-variable name, not its credential value.
|
// APIKeyEnv is an environment-variable name, not its credential value.
|
||||||
APIKeyEnv string `json:"api_key_env"`
|
APIKeyEnv string `json:"api_key_env"`
|
||||||
@@ -287,13 +313,15 @@ type ExecutionTarget struct {
|
|||||||
// ExecutionTargetOverride represents per-request runtime setting overrides and
|
// ExecutionTargetOverride represents per-request runtime setting overrides and
|
||||||
// has no stable JSON representation.
|
// has no stable JSON representation.
|
||||||
//
|
//
|
||||||
// Non-empty string fields replace profile values. Non-nil numeric pointers
|
// Non-empty string fields replace profile and backend values. Non-nil pointer
|
||||||
// replace profile values and preserve explicit zero. A non-empty ExtraParams
|
// fields replace profile values and preserve explicit zero or empty values. A
|
||||||
// map replaces the complete profile map rather than merging keys. Empty string
|
// non-empty ExtraParams map replaces the complete profile or backend map
|
||||||
// fields, nil pointers, and a nil or empty ExtraParams map inherit the selected
|
// rather than merging keys. Empty string fields, nil pointers, and a nil or
|
||||||
// profile over framework defaults.
|
// empty ExtraParams map inherit the selected profile over its backend, when
|
||||||
|
// any, and framework defaults.
|
||||||
type ExecutionTargetOverride struct {
|
type ExecutionTargetOverride struct {
|
||||||
// Endpoint replaces the profile endpoint when non-empty.
|
// Endpoint replaces the profile or backend endpoint when non-empty without
|
||||||
|
// changing the effective BackendID.
|
||||||
Endpoint string
|
Endpoint string
|
||||||
// Model replaces the profile model when non-empty.
|
// Model replaces the profile model when non-empty.
|
||||||
Model string
|
Model string
|
||||||
@@ -308,24 +336,29 @@ type ExecutionTargetOverride struct {
|
|||||||
TimeoutSeconds *int
|
TimeoutSeconds *int
|
||||||
// ServiceTier replaces the profile value when non-blank.
|
// ServiceTier replaces the profile value when non-blank.
|
||||||
ServiceTier string
|
ServiceTier string
|
||||||
// ReasoningEffort replaces the profile value when non-blank. An empty value
|
// ReasoningEffort controls the per-run reasoning setting. Nil inherits the
|
||||||
// cannot clear a profile setting.
|
// profile value. A pointer to a non-blank string trims and replaces the
|
||||||
ReasoningEffort string
|
// profile value. A pointer to an empty or whitespace-only string clears the
|
||||||
// APIKeyEnv replaces the profile environment-variable name when non-blank.
|
// inherited value and disables reasoning for this run. Non-blank values
|
||||||
// A direct RunRequest.APIKey still takes precedence over environment lookup.
|
// are opaque and are not validated against a fixed vocabulary.
|
||||||
|
ReasoningEffort *string
|
||||||
|
// APIKeyEnv replaces the profile or backend environment-variable name when
|
||||||
|
// non-blank. A direct RunRequest.APIKey still takes precedence over
|
||||||
|
// environment lookup.
|
||||||
APIKeyEnv string
|
APIKeyEnv string
|
||||||
// ExtraParams, when non-empty, replaces the profile map. Values must be
|
// ExtraParams, when non-empty, replaces the complete profile or backend map.
|
||||||
// JSON-compatible: nil, booleans, finite numbers, strings, arrays or slices,
|
// Values must be JSON-compatible: nil, booleans, finite numbers, strings,
|
||||||
// and maps with non-empty string keys. Cycles are invalid.
|
// arrays or slices, and maps with non-empty string keys. Cycles are invalid.
|
||||||
ExtraParams map[string]any
|
ExtraParams map[string]any
|
||||||
}
|
}
|
||||||
|
|
||||||
// Profile is an in-memory execution profile for library consumers.
|
// Profile is an in-memory execution profile for library consumers.
|
||||||
//
|
//
|
||||||
// It is equivalent to a loaded profile file after validation. Raw API keys do
|
// It is equivalent to a loaded profile file after validation. Raw API keys do
|
||||||
// not belong in profiles; use APIKeyRequired to require callers to provide
|
// not belong in profiles; use APIKeyRequired to require callers to provide a
|
||||||
// RunRequest.APIKey for each request, or use profile YAML api_key_env with file
|
// RunRequest.APIKey or explicit request ExecutionTargetOverride.APIKeyEnv, or
|
||||||
// and FS profile sources. Profile has no stable JSON representation.
|
// use profile YAML api_key_env with file and FS profile sources. Profile has no
|
||||||
|
// stable JSON representation.
|
||||||
//
|
//
|
||||||
// WithProfiles validates and copies Profile values during NewEngine. Numeric
|
// WithProfiles validates and copies Profile values during NewEngine. Numeric
|
||||||
// zero, blank strings, and an empty ExtraParams map inherit framework defaults;
|
// zero, blank strings, and an empty ExtraParams map inherit framework defaults;
|
||||||
@@ -333,7 +366,13 @@ type ExecutionTargetOverride struct {
|
|||||||
type Profile struct {
|
type Profile struct {
|
||||||
// ID is the required non-blank profile identifier. WithProfiles trims it.
|
// ID is the required non-blank profile identifier. WithProfiles trims it.
|
||||||
ID string
|
ID string
|
||||||
// Endpoint is the required non-blank model-provider base URL.
|
// BackendID optionally selects an engine backend. WithProfiles trims it.
|
||||||
|
// Backend membership is checked when a request selects the profile; an
|
||||||
|
// unknown ID makes preparation fail with ErrProfileLoad.
|
||||||
|
BackendID string
|
||||||
|
// Endpoint is the model-provider base URL. It is required only when
|
||||||
|
// BackendID is blank and otherwise overrides the backend endpoint when
|
||||||
|
// non-blank.
|
||||||
Endpoint string
|
Endpoint string
|
||||||
// Model is the required non-blank provider model identifier.
|
// Model is the required non-blank provider model identifier.
|
||||||
Model string
|
Model string
|
||||||
@@ -351,12 +390,13 @@ type Profile struct {
|
|||||||
// ReasoningEffort is optional; a blank value inherits the framework
|
// ReasoningEffort is optional; a blank value inherits the framework
|
||||||
// default.
|
// default.
|
||||||
ReasoningEffort string
|
ReasoningEffort string
|
||||||
// APIKeyRequired requires a non-blank RunRequest.APIKey. It does not store a
|
// APIKeyRequired clears a backend's inherited API-key environment name and
|
||||||
// credential or enable environment lookup.
|
// requires a non-blank RunRequest.APIKey unless the request explicitly
|
||||||
|
// supplies ExecutionTargetOverride.APIKeyEnv. It does not store a credential.
|
||||||
APIKeyRequired bool
|
APIKeyRequired bool
|
||||||
// ExtraParams contains provider-specific JSON-compatible values. An empty
|
// ExtraParams contains provider-specific JSON-compatible values. An empty
|
||||||
// map inherits framework defaults. WithProfiles validates and deeply copies
|
// map inherits backend request defaults, when any. WithProfiles validates
|
||||||
// it during NewEngine.
|
// and deeply copies it during NewEngine.
|
||||||
ExtraParams map[string]any
|
ExtraParams map[string]any
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -364,13 +404,15 @@ type Profile struct {
|
|||||||
// profile.
|
// profile.
|
||||||
//
|
//
|
||||||
// It contains ordinary profile fields for OpenAI-compatible chat-completions
|
// It contains ordinary profile fields for OpenAI-compatible chat-completions
|
||||||
// endpoints. APIKeyRequired is satisfied by RunRequest.APIKey. Raw API keys do
|
// endpoints. APIKeyRequired follows Profile.APIKeyRequired. Raw API keys do not
|
||||||
// not belong in this config. OpenAICompatibleProfileConfig has no stable JSON
|
// belong in this config. OpenAICompatibleProfileConfig has no stable JSON
|
||||||
// representation and is not validated until its resulting Profile is supplied
|
// representation and is not validated until its resulting Profile is supplied
|
||||||
// through WithProfiles to NewEngine.
|
// through WithProfiles to NewEngine.
|
||||||
type OpenAICompatibleProfileConfig struct {
|
type OpenAICompatibleProfileConfig struct {
|
||||||
// ID becomes Profile.ID.
|
// ID becomes Profile.ID.
|
||||||
ID string
|
ID string
|
||||||
|
// BackendID becomes Profile.BackendID.
|
||||||
|
BackendID string
|
||||||
// Endpoint becomes Profile.Endpoint.
|
// Endpoint becomes Profile.Endpoint.
|
||||||
Endpoint string
|
Endpoint string
|
||||||
// Model becomes Profile.Model.
|
// Model becomes Profile.Model.
|
||||||
@@ -470,8 +512,8 @@ type TokenUsage struct {
|
|||||||
// RenderedPrompt is the fully rendered prompt passed to an LLM client and has
|
// RenderedPrompt is the fully rendered prompt passed to an LLM client and has
|
||||||
// a stable JSON representation.
|
// a stable JSON representation.
|
||||||
type RenderedPrompt struct {
|
type RenderedPrompt struct {
|
||||||
// SessionID is the optional trimmed session identifier rendered from the
|
// SessionID is the optional effective direct or rendered session
|
||||||
// prompt definition.
|
// identifier supplied to the model client.
|
||||||
SessionID string `json:"session_id,omitempty"`
|
SessionID string `json:"session_id,omitempty"`
|
||||||
// Messages contains rendered messages in definition order.
|
// Messages contains rendered messages in definition order.
|
||||||
Messages []RenderedMessage `json:"messages"`
|
Messages []RenderedMessage `json:"messages"`
|
||||||
|
|||||||
Reference in New Issue
Block a user