Compare commits
15 Commits
v0.1.0
...
ffe6d261a9
| Author | SHA1 | Date | |
|---|---|---|---|
| ffe6d261a9 | |||
| dc39562ff7 | |||
| 0a839aa16d | |||
| f6ee18f6b3 | |||
| eb8ab215e8 | |||
| f89cb94ed2 | |||
| 359b7313f4 | |||
| ae210b3c26 | |||
| 810f80e7c9 | |||
| 8d00354c59 | |||
| d0010689f3 | |||
| b462153483 | |||
| 086cf0fc86 | |||
| c1cecb1ee8 | |||
| bcb327f643 |
@@ -31,4 +31,9 @@ Contributors should start with the [development guide](docs/development.md).
|
||||
The [architecture policy](docs/policy/architecture.md) defines the library
|
||||
boundary and constraints that framework work must preserve.
|
||||
|
||||
## Related Project
|
||||
|
||||
[Scriptorium](https://gitea.maximumdirect.net/eric/scriptorium) is the CLI and
|
||||
HTTP application built on Promptkit.
|
||||
|
||||
Promptkit is licensed under the [GNU General Public License version 3](LICENSE).
|
||||
|
||||
55
backends.go
Normal file
55
backends.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
// BackendOpenRouter is the reserved ID of Promptkit's built-in OpenRouter
|
||||
// backend.
|
||||
const BackendOpenRouter = backend.OpenRouterID
|
||||
|
||||
// Backend configures one engine-scoped OpenAI-compatible backend.
|
||||
//
|
||||
// Backend has no stable JSON representation. Use keyed literals so additions
|
||||
// to this configuration value do not break source compatibility.
|
||||
type Backend struct {
|
||||
// ID is the stable, case-sensitive registry key. NewEngine trims it and
|
||||
// requires a non-blank value. BackendOpenRouter is reserved.
|
||||
ID string
|
||||
// Endpoint is the OpenAI-compatible base endpoint. NewEngine trims it and
|
||||
// requires an absolute HTTP or HTTPS URL with a host and without user
|
||||
// information, a query string, or a fragment. Paths are allowed.
|
||||
Endpoint string
|
||||
// APIKeyEnv optionally names the environment variable containing the API
|
||||
// key. NewEngine trims it and requires the portable form
|
||||
// [A-Za-z_][A-Za-z0-9_]*. Store only the name, never a credential value.
|
||||
APIKeyEnv string
|
||||
// ExtraParams contains backend-wide request defaults. Values must be
|
||||
// JSON-compatible, finite, acyclic, and keyed by non-empty strings. Keys
|
||||
// must not be model, session_id, messages, temperature, max_tokens, top_p,
|
||||
// service_tier, reasoning_effort, or response_format. An empty map supplies
|
||||
// no defaults. NewEngine deeply copies the map.
|
||||
ExtraParams map[string]any
|
||||
}
|
||||
|
||||
// WithBackend adds one Backend registration to the constructed Engine.
|
||||
//
|
||||
// Registrations accumulate in option order. Every normalized ID must be unique
|
||||
// across consumer registrations and built-ins; a duplicate or invalid
|
||||
// definition makes NewEngine fail with ErrInvalidConfig. In particular,
|
||||
// BackendOpenRouter cannot be replaced. The immutable registration is scoped
|
||||
// to the resulting Engine and cannot be enumerated, replaced, removed, or
|
||||
// mutated after construction. WithBackend does not install package-global
|
||||
// state.
|
||||
func WithBackend(backend Backend) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
options.backends = append(options.backends, domain.Backend{
|
||||
ID: backend.ID,
|
||||
Endpoint: backend.Endpoint,
|
||||
APIKeyEnv: backend.APIKeyEnv,
|
||||
ExtraParams: backend.ExtraParams,
|
||||
})
|
||||
return nil
|
||||
})
|
||||
}
|
||||
19
convert.go
19
convert.go
@@ -4,6 +4,7 @@ import (
|
||||
"reflect"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||
)
|
||||
|
||||
func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
||||
@@ -15,12 +16,12 @@ func toDomainRunRequest(req RunRequest) (domain.RunRequest, error) {
|
||||
PromptID: req.PromptID,
|
||||
PromptVersion: req.PromptVersion,
|
||||
ProfileID: req.ProfileID,
|
||||
SessionID: req.SessionID,
|
||||
APIKey: req.APIKey,
|
||||
Inputs: toDomainArtifactRefMap(req.Inputs),
|
||||
Vars: copyStringMap(req.Vars),
|
||||
Execution: execution,
|
||||
Validation: toDomainOutputContractPtr(req.Validation),
|
||||
Metadata: copyStringMap(req.Metadata),
|
||||
}, nil
|
||||
}
|
||||
|
||||
@@ -33,6 +34,7 @@ func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
|
||||
PromptVersion: prepared.PromptVersion,
|
||||
PromptHash: prepared.PromptHash,
|
||||
SelectedProfileID: prepared.SelectedProfileID,
|
||||
SelectedBackendID: prepared.SelectedBackendID,
|
||||
EffectiveModelParams: fromDomainExecutionTarget(prepared.EffectiveModelParams),
|
||||
OutputContract: fromDomainOutputContract(prepared.OutputContract),
|
||||
StructuredOutput: fromDomainStructuredOutputSpec(prepared.StructuredOutput),
|
||||
@@ -58,8 +60,10 @@ func fromDomainRunResult(result *domain.RunResult) *RunResult {
|
||||
PromptID: result.PromptID,
|
||||
PromptVersion: result.PromptVersion,
|
||||
PromptHash: result.PromptHash,
|
||||
SessionID: result.SessionID,
|
||||
RenderedPromptHash: result.RenderedPromptHash,
|
||||
SelectedProfileID: result.SelectedProfileID,
|
||||
SelectedBackendID: result.SelectedBackendID,
|
||||
ModelName: result.ModelName,
|
||||
Endpoint: result.Endpoint,
|
||||
EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams),
|
||||
@@ -132,7 +136,7 @@ func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain
|
||||
if override == nil {
|
||||
return nil, nil
|
||||
}
|
||||
extraParams, err := copyPublicJSONMap(override.ExtraParams)
|
||||
extraParams, err := jsonvalue.CopyMap(override.ExtraParams)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -144,7 +148,7 @@ func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain
|
||||
TopP: copyFloat64Ptr(override.TopP),
|
||||
TimeoutSeconds: copyIntPtr(override.TimeoutSeconds),
|
||||
ServiceTier: override.ServiceTier,
|
||||
ReasoningEffort: override.ReasoningEffort,
|
||||
ReasoningEffort: copyStringPtr(override.ReasoningEffort),
|
||||
APIKeyEnv: override.APIKeyEnv,
|
||||
ExtraParams: extraParams,
|
||||
}, nil
|
||||
@@ -152,6 +156,7 @@ func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain
|
||||
|
||||
func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
|
||||
return ExecutionTarget{
|
||||
BackendID: target.BackendID,
|
||||
Endpoint: target.Endpoint,
|
||||
Model: target.Model,
|
||||
Temperature: target.Temperature,
|
||||
@@ -397,6 +402,14 @@ func copyFloat64Ptr(src *float64) *float64 {
|
||||
return &v
|
||||
}
|
||||
|
||||
func copyStringPtr(src *string) *string {
|
||||
if src == nil {
|
||||
return nil
|
||||
}
|
||||
v := *src
|
||||
return &v
|
||||
}
|
||||
|
||||
func copyIntPtr(src *int) *int {
|
||||
if src == nil {
|
||||
return nil
|
||||
|
||||
55
doc.go
55
doc.go
@@ -1,8 +1,55 @@
|
||||
// Package promptkit provides an embeddable engine for preparing and executing
|
||||
// prompt-defined LLM workflows.
|
||||
//
|
||||
// Applications construct an Engine with NewEngine, select filesystem or
|
||||
// in-memory definition sources with options, and use Prepare or Run to execute
|
||||
// requests. Concrete repositories, validators, and outbound clients remain
|
||||
// internal implementation details.
|
||||
// Applications construct an [Engine] with [NewEngine], select filesystem or
|
||||
// in-memory sources and optional engine-scoped [Backend] registrations, and
|
||||
// call [Engine.Prepare] or [Engine.Run]. Concrete registries, repositories,
|
||||
// validators, and the built-in OpenAI-compatible client remain internal
|
||||
// implementation details.
|
||||
//
|
||||
// # Concurrency and ownership
|
||||
//
|
||||
// An Engine supports concurrent Prepare and Run calls. An injected [LLMClient]
|
||||
// or [ArtifactReader] can therefore receive concurrent calls and must be safe
|
||||
// for that use.
|
||||
//
|
||||
// NewEngine copies in-memory profiles and backend definitions. Prepare and Run
|
||||
// copy request maps, slices, pointer values, and JSON-compatible extra
|
||||
// parameters before using them. Returned values and values passed to extension
|
||||
// interfaces are likewise isolated from engine state. Callers own those copies
|
||||
// and may mutate them after the call that supplied or returned them.
|
||||
//
|
||||
// # Security and sensitive data
|
||||
//
|
||||
// The default artifact reader treats [File] paths as caller-selected operating
|
||||
// system paths. It does not restrict them to an application root or impose an
|
||||
// inbound request-size policy. Promptkit is not an inbound request or
|
||||
// untrusted-input security boundary. Applications must validate and restrict
|
||||
// untrusted input before constructing a request, or install an [ArtifactReader]
|
||||
// that enforces their filesystem, authorization, and size policies.
|
||||
//
|
||||
// Rendered messages, input and output [Artifact] bodies, [RunResult.RawOutput],
|
||||
// and [ValidationResult.Errors] may contain sensitive data. Credential
|
||||
// exclusion and redaction do not sanitize those values. Applications and
|
||||
// injected collaborators are responsible for access control, retention,
|
||||
// logging, and secret handling appropriate to their data.
|
||||
//
|
||||
// # JSON
|
||||
//
|
||||
// Stable JSON representations are provided for [PreparedRun], [RunResult],
|
||||
// [Artifact], [ExecutionTarget], [OutputContract], [ValidationResult],
|
||||
// [TokenUsage], [RenderedPrompt], [RenderedMessage], [CacheControl],
|
||||
// [StructuredOutputSpec], [StructuredOutputJSONSpec], [GenerateRequest],
|
||||
// [GenerateResponse], [ExecutionTargetPresence], and the string value types
|
||||
// used by those values.
|
||||
//
|
||||
// Construction values, including [Config], [Backend], [RunRequest],
|
||||
// [ArtifactRef], [ExecutionTargetOverride], [Profile], and
|
||||
// [OpenAICompatibleProfileConfig], do not have stable JSON representations.
|
||||
// Direct API keys are nevertheless excluded from JSON for every public value.
|
||||
//
|
||||
// JSON timestamps use time.Time's RFC 3339 encoding and are omitted when zero.
|
||||
// PreparedRun and RunResult durations are encoded as integer milliseconds in
|
||||
// duration_ms and omitted when zero. Run IDs and all exposed hashes are opaque:
|
||||
// their spelling, length, character set, and algorithm are not API contracts.
|
||||
package promptkit
|
||||
|
||||
@@ -1,151 +1,208 @@
|
||||
# Package `promptkit`
|
||||
|
||||
Import path:
|
||||
## Purpose
|
||||
|
||||
This guide helps Go consumers assemble Promptkit and choose the main
|
||||
preparation or execution workflow. The declarations and GoDoc in the
|
||||
[root package](../../doc.go) own exact field, option, serialization,
|
||||
concurrency, ownership, failure, and cancellation semantics. The
|
||||
[framework format reference](../formats.md) owns prompt, profile, and schema
|
||||
file contracts.
|
||||
|
||||
Import the package as:
|
||||
|
||||
```go
|
||||
import "gitea.maximumdirect.net/eric/promptkit"
|
||||
```
|
||||
|
||||
Package `promptkit` is the supported Go contract for in-process prompt
|
||||
preparation and execution. The declarations and their GoDoc in the
|
||||
[root package](../../doc.go) own the exact API; this guide explains how the
|
||||
pieces are used together. The [framework format reference](../formats.md) owns
|
||||
prompt, profile, and schema file contracts.
|
||||
The following Go fragments are illustrative and omit surrounding package,
|
||||
import, and error-handling code. Use the maintained examples for complete
|
||||
programs.
|
||||
|
||||
## Engine Construction And Sources
|
||||
## Construct An Engine
|
||||
|
||||
Construct an engine with [`NewEngine`, `Config`, and
|
||||
`Option`](../../engine.go). `PromptDir` is required unless a prompt source
|
||||
option is supplied. `ProfileDir` optionally overlays built-in profiles, and an
|
||||
empty `SchemaDir` uses the current directory. `Timeout` is the transport-wide
|
||||
safety cap for the built-in OpenAI-compatible client. An optional `HTTPClient`
|
||||
is cloned; its positive timeout takes precedence.
|
||||
Create an engine with
|
||||
[`NewEngine`](../../engine.go). A directory-backed setup supplies a prompt
|
||||
directory and may supply profile and schema directories:
|
||||
|
||||
Nil options are ignored. Invalid construction, including a nil injected client
|
||||
or artifact reader, returns an error matching `ErrInvalidConfig`.
|
||||
```go
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
PromptDir: "prompts",
|
||||
ProfileDir: "profiles",
|
||||
SchemaDir: "schemas",
|
||||
})
|
||||
```
|
||||
|
||||
The [source options](../../engine.go) replace their matching directory source:
|
||||
|
||||
- `WithPromptFS` and `WithPromptFile` select prompt definitions;
|
||||
- `WithProfileFS` and `WithProfileFile` overlay built-in profiles;
|
||||
- `WithProfiles` adds in-memory profiles ahead of file and built-in profiles;
|
||||
- `WithSchemaFS` and `WithSchemaFile` select JSON Schema documents;
|
||||
- `WithLLMClient` replaces the built-in model client; and
|
||||
- `WithArtifactReader` replaces the default reader for every input.
|
||||
|
||||
Source selection, path resolution, strict decoding, profile overlays, and
|
||||
file-to-request precedence are defined in the
|
||||
Options support single-file or `fs.FS` sources, in-memory profiles,
|
||||
engine-scoped backends, and injected artifact or model clients. Consult the
|
||||
[constructor and option GoDoc](../../engine.go) for composition, precedence,
|
||||
validation, and default transport behavior. Source discovery, format
|
||||
validation, and profile precedence are defined by the
|
||||
[framework format reference](../formats.md).
|
||||
|
||||
Per-generation timeout values from profiles or requests are independent of
|
||||
the transport cap and caller context. An explicit request value of zero
|
||||
disables only the per-generation deadline. The
|
||||
[outbound integration contract](../integrations/openai-compatible-chat.md#timeout-and-cancellation)
|
||||
defines the complete timeout layering.
|
||||
## Prepare Without Model Execution
|
||||
|
||||
## Preparation And Execution
|
||||
[`Engine.Prepare`](../../engine.go) resolves the selected prompt and profile,
|
||||
loads inputs and any structured-output schema, and renders messages without
|
||||
calling a model client:
|
||||
|
||||
[`Engine.Prepare` and `Engine.Run`](../../engine.go) accept the public
|
||||
[`RunRequest`](../../types.go). `Prepare` resolves the prompt, profile, input
|
||||
artifacts, validation contract, and rendered messages without calling an LLM.
|
||||
`Run` performs the same preparation, calls the configured client, and validates
|
||||
the generated content. The maintained
|
||||
```go
|
||||
prepared, err := engine.Prepare(ctx, promptkit.RunRequest{
|
||||
PromptID: "meeting.summary",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"note": promptkit.Inline("Synthetic meeting notes"),
|
||||
},
|
||||
})
|
||||
```
|
||||
|
||||
The maintained
|
||||
[offline preparation example](../../examples/go-library/prepare/main.go)
|
||||
provides a complete runnable workflow using a prompt file, in-memory profile,
|
||||
and inline input.
|
||||
shows a complete runnable setup with a prompt file, in-memory profile, and
|
||||
inline input. Exact request requirements and prepared-result fields belong to
|
||||
the [`RunRequest` and `PreparedRun` GoDoc](../../types.go).
|
||||
|
||||
[`PreparedRun` and `RunResult`](../../types.go) expose copied public values.
|
||||
Preparation returns effective settings, hashes, rendered messages, selected
|
||||
profile, structured-output information, and timing without resolved secrets or
|
||||
model output. Execution adds the generated artifact and raw output, validation
|
||||
state, model metadata, usage, run ID, and duration.
|
||||
## Execute And Validate
|
||||
|
||||
A generated-content validation failure returns a result with
|
||||
`Validation.Status == ValidationFailed`. An inability to perform validation
|
||||
returns an error matching `ErrValidation`.
|
||||
[`Engine.Run`](../../engine.go) performs the same preparation, invokes the
|
||||
configured model client, classifies the generated artifact, and validates the
|
||||
content. A completed content check may return `ValidationFailed` in the result;
|
||||
an operational inability to validate returns an error.
|
||||
|
||||
## Requests, Inputs, And Overrides
|
||||
The maintained
|
||||
[offline execution example](../../examples/go-library/run/main.go) injects a
|
||||
deterministic model client and exercises `Run` without credentials, network
|
||||
access, or paid calls. It is intentionally separate from the preparation
|
||||
example so each workflow and its small prompt fixture can be copied and run on
|
||||
its own.
|
||||
|
||||
The [request and value declarations](../../types.go) own the available fields,
|
||||
serialized constants, and result shapes. Use `File`, `Inline`, or
|
||||
`InlineWithURI` to construct artifact references. The
|
||||
[framework format reference](../formats.md) defines declared inputs, template
|
||||
references, output contracts, and the relationship between file values and
|
||||
request overrides.
|
||||
Use the [`RunResult` and `ValidationResult` GoDoc](../../types.go) for the
|
||||
returned data and the `Engine.Run` GoDoc for failure and cancellation
|
||||
semantics. The
|
||||
[OpenAI-compatible integration contract](../integrations/openai-compatible-chat.md)
|
||||
owns the built-in client's outbound HTTP behavior.
|
||||
|
||||
`ExecutionTargetOverride` uses pointers for numeric settings so an explicit
|
||||
zero remains distinct from no override. `ExtraParams` accepts JSON-compatible
|
||||
strings, booleans, finite numbers, string-keyed objects, arrays or slices, and
|
||||
nil. Unsupported values, non-string map keys, non-finite numbers, and cycles
|
||||
match `ErrInvalidConfig` in profiles or `ErrInvalidRequest` in request
|
||||
overrides.
|
||||
## Inputs, Profiles, And Overrides
|
||||
|
||||
Returned requests, profiles, prepared values, results, artifacts, maps, and
|
||||
slices are isolated from internal engine state. Consumers and injected
|
||||
extensions should not retain or mutate values owned by another caller.
|
||||
Use `File`, `Inline`, or `InlineWithURI` to construct request inputs. A request
|
||||
can select a profile explicitly or use the prompt's default profile, and can
|
||||
replace execution settings or the complete output contract.
|
||||
|
||||
## Profiles And Credentials
|
||||
The [public value GoDoc](../../types.go) defines nil, empty, zero, replacement,
|
||||
copy, and credential behavior. The
|
||||
[framework format reference](../formats.md) defines how those request values
|
||||
interact with prompt definitions, file-backed profiles, built-ins, schemas,
|
||||
and framework defaults.
|
||||
|
||||
[`OpenAICompatibleProfile`](../../profiles.go) constructs an ordinary
|
||||
in-memory profile for an OpenAI-compatible chat-completions endpoint.
|
||||
`WithProfiles` rejects duplicate IDs in one call and gives in-memory profiles
|
||||
precedence over explicit file sources and built-ins.
|
||||
For programmatic profiles,
|
||||
[`OpenAICompatibleProfile`](../../profiles.go) converts ordinary
|
||||
OpenAI-compatible settings into a value accepted by `WithProfiles`.
|
||||
|
||||
Raw API keys do not belong in profiles. File-backed profiles may name an
|
||||
environment variable, while an in-memory profile can require a request key.
|
||||
A direct `RunRequest.APIKey` is request-scoped and takes precedence over an
|
||||
environment lookup for the built-in client. Profile fields, ranges, built-ins,
|
||||
precedence, and credential rules are owned by the
|
||||
[framework format reference](../formats.md).
|
||||
### Set A Per-Run Session And Reasoning
|
||||
|
||||
API keys are excluded from JSON, prepared values, and results. The public
|
||||
`String` and `GoString` methods report only whether a direct key is present.
|
||||
Avoid reflection-based dumps of request structs, which can bypass that
|
||||
redaction.
|
||||
Supply a direct session ID when one prompt should be correlated with a
|
||||
consumer-managed conversation or workflow without changing prompt variables:
|
||||
|
||||
```go
|
||||
reasoning := "high"
|
||||
result, err := engine.Run(ctx, promptkit.RunRequest{
|
||||
PromptID: "meeting.summary",
|
||||
SessionID: "conversation-42",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"note": promptkit.Inline("Synthetic meeting notes"),
|
||||
},
|
||||
Execution: &promptkit.ExecutionTargetOverride{
|
||||
ReasoningEffort: &reasoning,
|
||||
},
|
||||
})
|
||||
```
|
||||
|
||||
A nil reasoning pointer inherits the selected profile, a pointer to a
|
||||
nonblank string replaces it, and a pointer to a blank string disables
|
||||
reasoning for that run. Session IDs are correlation metadata, not credentials;
|
||||
use stable, non-secret values that are safe to expose to collaborators and
|
||||
providers. The
|
||||
[`RunRequest` and `ExecutionTargetOverride` GoDoc](../../types.go) owns the
|
||||
exact normalization, precedence, error, copying, and exposure contract.
|
||||
|
||||
### Register A Custom Backend
|
||||
|
||||
Register a reusable OpenAI-compatible connection once, then select it from a
|
||||
profile:
|
||||
|
||||
```go
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
PromptDir: "prompts",
|
||||
},
|
||||
promptkit.WithBackend(promptkit.Backend{
|
||||
ID: "local",
|
||||
Endpoint: "http://localhost:8000/v1",
|
||||
APIKeyEnv: "LOCAL_LLM_API_KEY",
|
||||
}),
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "local-summary",
|
||||
BackendID: "local",
|
||||
Model: "example-model",
|
||||
}),
|
||||
)
|
||||
```
|
||||
|
||||
Registrations belong to one engine and custom IDs cannot replace built-ins.
|
||||
The [`Backend` and `WithBackend` GoDoc](../../backends.go) defines validation,
|
||||
copying, uniqueness, and request-default behavior.
|
||||
|
||||
Both file-backed and in-memory profiles select a registration through
|
||||
`backend` or `Profile.BackendID`. Profile and request endpoint overrides retain
|
||||
that routing identity. `PreparedRun.SelectedBackendID`,
|
||||
`RunResult.SelectedBackendID`, and the effective `ExecutionTarget.BackendID`
|
||||
expose it to consumers and injected model clients. Endpoint-only profiles
|
||||
remain supported and expose an empty backend ID.
|
||||
|
||||
## Credentials
|
||||
|
||||
File-backed profiles name an environment variable; in-memory profiles can
|
||||
require a direct request key. Direct keys are request-scoped and are excluded
|
||||
from supported JSON values and the package's `String` and `GoString`
|
||||
summaries. The exact precedence and redaction guarantees belong to
|
||||
[`RunRequest`, `GenerateRequest`, and the profile GoDoc](../../types.go).
|
||||
|
||||
## Protect Files And Generated Data
|
||||
|
||||
The default artifact reader opens a `File` reference as a caller-selected
|
||||
operating-system path. It does not constrain paths to an application root,
|
||||
impose an inbound request-size policy, or establish an untrusted-input security
|
||||
boundary. Applications must validate and restrict untrusted paths and payloads
|
||||
before constructing a request, or inject an artifact reader that enforces
|
||||
their filesystem, authorization, and size policies.
|
||||
|
||||
Rendered messages, input and output artifact bodies, raw model output, and
|
||||
validation diagnostics can contain sensitive data. API-key redaction does not
|
||||
sanitize those values. Treat prepared values, results, collaborator requests,
|
||||
errors, and logs according to the application's data-access, retention, and
|
||||
secret-handling policies.
|
||||
|
||||
## Extension Interfaces
|
||||
|
||||
The [`LLMClient`, `GenerateRequest`, and
|
||||
`GenerateResponse`](../../types.go) boundary lets a consumer replace model
|
||||
generation. Injected clients receive copied rendered messages, effective
|
||||
settings, explicit numeric-setting presence, structured-output constraints,
|
||||
and the request-scoped key. They return generated content and token usage.
|
||||
Inject an [`LLMClient` or `ArtifactReader`](../../types.go) when the built-in
|
||||
behavior does not fit the application. Their GoDoc defines concurrent use,
|
||||
context handling, ownership of copied values, nil responses, and preservation
|
||||
of collaborator errors. Implementations must honor cancellation, safely manage
|
||||
copies they retain, avoid unsafe logging of content or credentials, and enforce
|
||||
the application policy that motivated the injection.
|
||||
|
||||
The [`ArtifactReader`](../../types.go) boundary replaces the default inline and
|
||||
file reader for every input. Readers provide artifact content and metadata; the
|
||||
engine fills an empty artifact name from the input-map key. A reader error
|
||||
matches `ErrArtifactLoad` while preserving the original identity for
|
||||
`errors.Is`. A nil artifact with a nil error is also an artifact-load failure.
|
||||
## Handle Errors
|
||||
|
||||
Extensions should honor context cancellation and avoid logging raw prompts,
|
||||
artifacts, or credentials.
|
||||
Use `errors.Is` with the
|
||||
[public error sentinels and operation GoDoc](../../engine.go). The declarations
|
||||
distinguish invalid construction, invalid requests, absent sources,
|
||||
source-loading failures, collaborator failures, and operational validation
|
||||
failures. Specific request conditions may also match the broader
|
||||
`ErrInvalidRequest`, and injected collaborator identities are preserved where
|
||||
documented. Invalid or duplicate backend registrations match
|
||||
`ErrInvalidConfig`; selecting an unknown backend matches `ErrProfileLoad`.
|
||||
|
||||
## Errors
|
||||
|
||||
The [public error declarations](../../engine.go) and
|
||||
[mapping](../../errors.go) preserve these sentinel checks through `errors.Is`:
|
||||
|
||||
- `ErrInvalidConfig`
|
||||
- `ErrInvalidRequest`
|
||||
- `ErrPromptNotFound`
|
||||
- `ErrProfileNotFound`
|
||||
- `ErrProfileRequired`
|
||||
- `ErrPromptLoad`
|
||||
- `ErrProfileLoad`
|
||||
- `ErrAPIKeyEnvMissing`
|
||||
- `ErrArtifactLoad`
|
||||
- `ErrPromptRender`
|
||||
- `ErrLLMGenerate`
|
||||
- `ErrValidation`
|
||||
|
||||
`ErrProfileRequired` and `ErrAPIKeyEnvMissing` also match
|
||||
`ErrInvalidRequest`, allowing either broad request handling or a specific
|
||||
condition. Wrapped collaborator errors retain their identity where the public
|
||||
contract promises it.
|
||||
|
||||
## Consumer Boundary
|
||||
## Application Boundary
|
||||
|
||||
Promptkit is an importable library. It does not own a command, inbound HTTP
|
||||
API, process configuration, or deployment policy. Scriptorium is one
|
||||
downstream application that maps this root package contract into those
|
||||
application concerns.
|
||||
API, process configuration, or deployment policy. Applications map the root
|
||||
package's results and errors into those concerns, including inbound size and
|
||||
trust policy.
|
||||
|
||||
@@ -91,7 +91,8 @@ of a named input. Missing variables and input references are errors.
|
||||
|
||||
The optional `session_id` uses the same template data and input helper. Its
|
||||
rendered value is trimmed, omitted when empty, and limited to 256 Unicode code
|
||||
points.
|
||||
points. A nonblank direct request session ID bypasses this template completely;
|
||||
a blank direct value leaves the template behavior unchanged.
|
||||
|
||||
### Cache Control
|
||||
|
||||
@@ -136,7 +137,7 @@ A profile supplies model execution settings:
|
||||
|
||||
```yaml
|
||||
id: local-summary
|
||||
endpoint: http://localhost:8000/v1
|
||||
backend: openrouter
|
||||
model: example-model
|
||||
temperature: 0.2
|
||||
max_tokens: 500
|
||||
@@ -144,7 +145,6 @@ top_p: 0.95
|
||||
timeout_seconds: 90
|
||||
service_tier: flex
|
||||
reasoning_effort: medium
|
||||
api_key_env: EXAMPLE_API_KEY
|
||||
extra_params:
|
||||
provider_option: enabled
|
||||
```
|
||||
@@ -152,7 +152,8 @@ extra_params:
|
||||
| Field | Required | Meaning |
|
||||
| --- | --- | --- |
|
||||
| `id` | yes | Non-empty profile identifier. IDs must be unique within one source. |
|
||||
| `endpoint` | yes | Non-empty OpenAI-compatible base URL, including an API version path when required. |
|
||||
| `backend` | unless `endpoint` is present | Backend registry ID. It is trimmed and registry membership is checked when the profile is prepared. |
|
||||
| `endpoint` | unless `backend` is present | Non-empty OpenAI-compatible base URL, including an API version path when required. When both connection fields are present, this overrides the backend endpoint without changing backend identity. |
|
||||
| `model` | yes | Non-empty provider model name. |
|
||||
| `temperature` | no | Number from 0 through 2. |
|
||||
| `max_tokens` | no | Integer zero or greater. |
|
||||
@@ -166,6 +167,13 @@ extra_params:
|
||||
Raw `api_key` is prohibited in profile YAML. Store only an environment
|
||||
variable name in `api_key_env`.
|
||||
|
||||
Promptkit does not infer a backend from a model or endpoint. Endpoint-only
|
||||
profiles remain supported and have no effective backend ID.
|
||||
The engine always provides the built-in `openrouter` ID. Consumers can add
|
||||
engine-scoped IDs with
|
||||
[`WithBackend`](../backends.go); exact registration validation belongs to its
|
||||
GoDoc.
|
||||
|
||||
`extra_params` accepts null, booleans, finite numbers, strings, arrays, and
|
||||
objects with string keys. Keys must be non-empty. With the built-in client,
|
||||
they also cannot collide with the standard fields listed in the
|
||||
@@ -176,8 +184,9 @@ they also cannot collide with the standard fields listed in the
|
||||
Execution settings resolve in this order:
|
||||
|
||||
1. framework defaults;
|
||||
2. the selected profile; and
|
||||
3. request `ExecutionTargetOverride` values.
|
||||
2. the selected backend, when the profile names one;
|
||||
3. the selected profile; and
|
||||
4. request `ExecutionTargetOverride` values.
|
||||
|
||||
The framework defaults are:
|
||||
|
||||
@@ -194,8 +203,13 @@ explicit zero is preserved. In particular, an explicit request
|
||||
`timeout_seconds` of zero disables the per-generation deadline while leaving
|
||||
the caller context and transport timeout intact.
|
||||
|
||||
Non-empty request strings replace profile strings. A non-empty request
|
||||
`ExtraParams` map replaces the profile map rather than merging keys.
|
||||
Non-empty profile strings replace backend defaults, and non-empty request
|
||||
strings replace both. Request reasoning is the exception: a nil
|
||||
`ReasoningEffort` pointer inherits the profile, a pointer to a nonblank string
|
||||
trims and replaces it, and a pointer to a blank string clears it. Backend
|
||||
identity is retained when either layer overrides the endpoint. A non-empty
|
||||
`extra_params` map at each layer replaces the entire lower-precedence map
|
||||
rather than merging keys.
|
||||
The [outbound integration contract](integrations/openai-compatible-chat.md)
|
||||
defines how the effective settings are serialized.
|
||||
|
||||
@@ -217,8 +231,11 @@ invalid matching profile is an error and does not fall back. In-memory
|
||||
|
||||
## Built-In Profile Catalog
|
||||
|
||||
Built-ins use the OpenRouter-compatible endpoint and
|
||||
`OPENROUTER_API_KEY`. A custom or in-memory profile with the same ID takes
|
||||
Every built-in selects the `openrouter` backend. The engine's built-in backend
|
||||
registry supplies `https://openrouter.ai/api/v1` and the environment-variable
|
||||
name `OPENROUTER_API_KEY`, so individual profiles contain only model and
|
||||
generation settings. Built-in profile files do not repeat those connection
|
||||
values. A custom or in-memory profile with the same profile ID takes
|
||||
precedence.
|
||||
|
||||
| Provider | ID | Model |
|
||||
@@ -270,6 +287,10 @@ prompt, profile, schema, or example files:
|
||||
- a request can provide a direct `APIKey` or override `APIKeyEnv`; and
|
||||
- a direct request key takes precedence over environment lookup.
|
||||
|
||||
After a direct request key, the credential-source precedence is request
|
||||
`APIKeyEnv`, profile `api_key_env`, then the backend default. An in-memory
|
||||
profile with `APIKeyRequired` clears an inherited backend environment name and
|
||||
requires a direct key unless the request explicitly supplies `APIKeyEnv`.
|
||||
Promptkit validates required credential availability during preparation.
|
||||
Direct keys are excluded from JSON results and redacted by public string
|
||||
formatters. Environment-variable names may appear in prepared metadata, but
|
||||
|
||||
@@ -13,10 +13,15 @@ that produce these outbound settings.
|
||||
## Endpoint And Method
|
||||
|
||||
Generation sends an HTTP `POST` with `Content-Type: application/json`.
|
||||
A non-empty endpoint from the execution target overrides the client's
|
||||
configured base URL. After trailing slashes are removed,
|
||||
`/chat/completions` is appended. Generation fails before sending when neither
|
||||
source supplies an endpoint.
|
||||
Before the client is called, the engine resolves framework, backend, profile,
|
||||
and request values into one execution target. A non-empty endpoint from that
|
||||
target overrides the client's configured base URL. After trailing slashes are
|
||||
removed, `/chat/completions` is appended. Generation fails before sending when
|
||||
neither source supplies an endpoint.
|
||||
|
||||
The target's backend ID is routing metadata for prepared values, results, and
|
||||
injected clients. The built-in client does not derive the URL from that ID and
|
||||
does not serialize it in the provider request.
|
||||
|
||||
## Authentication
|
||||
|
||||
@@ -26,6 +31,12 @@ the client reads that variable and requires a non-empty value. The selected
|
||||
key is sent as `Authorization: Bearer <key>`. No authorization header is sent
|
||||
when neither mechanism is configured.
|
||||
|
||||
The target contains the already resolved environment-variable name: an
|
||||
explicit request override takes precedence over profile metadata, which takes
|
||||
precedence over the backend default. Only the name reaches prepared metadata;
|
||||
the environment value is read just before the provider call and is never added
|
||||
to the JSON body.
|
||||
|
||||
## Request Body
|
||||
|
||||
The request body always contains `model` and `messages`. The execution
|
||||
@@ -36,20 +47,24 @@ Each ordinary message contains its `role` and string `content`. A
|
||||
cache-controlled message instead uses a text content block containing `type`,
|
||||
`text`, and `cache_control`; an empty cache-control TTL is omitted.
|
||||
|
||||
A non-empty session ID is trimmed, checked against the internal domain limit,
|
||||
and sent as top-level `session_id`. It is not sent as a session header.
|
||||
The effective direct or prompt-rendered session ID is trimmed, limited to 256
|
||||
Unicode code points, and sent when nonempty as top-level `session_id`. It is
|
||||
never also sent as a session header.
|
||||
|
||||
The client conditionally includes:
|
||||
|
||||
- `temperature`, `max_tokens`, and `top_p` when non-zero or explicitly
|
||||
present;
|
||||
- non-empty `service_tier` and `reasoning_effort`; and
|
||||
- non-empty `service_tier` and effective `reasoning_effort`; an explicitly
|
||||
disabled reasoning setting is empty and therefore omitted; and
|
||||
- `response_format` for JSON Schema structured output, including its name,
|
||||
strict flag, and schema document.
|
||||
|
||||
Extra parameters are merged directly into the top-level body after JSON
|
||||
serialization is verified. Empty keys and collisions with these reserved
|
||||
fields are rejected before any provider call:
|
||||
The engine resolves backend, profile, and request extra-parameter maps by
|
||||
whole-map replacement rather than key merging. The resulting effective map is
|
||||
then merged directly into the top-level body after JSON serialization is
|
||||
verified. Empty keys and collisions with these reserved fields are rejected
|
||||
before any provider call:
|
||||
|
||||
- `model`
|
||||
- `session_id`
|
||||
@@ -61,6 +76,9 @@ fields are rejected before any provider call:
|
||||
- `reasoning_effort`
|
||||
- `response_format`
|
||||
|
||||
`backend_id`, `api_key_env`, and resolved credential values are not provider
|
||||
request fields.
|
||||
|
||||
## Response Handling
|
||||
|
||||
Any 2xx response is decoded as an OpenAI-compatible chat response. The client
|
||||
|
||||
@@ -20,6 +20,11 @@ orchestration. `OpenAICompatibleClient` is the built-in implementation. It
|
||||
uses internal domain values for rendered prompts, execution targets,
|
||||
structured output, responses, and token usage.
|
||||
|
||||
The runner supplies a fully resolved target after applying backend, profile,
|
||||
and request precedence. The client uses its endpoint, credential metadata,
|
||||
generation fields, and extra parameters. `BackendID` remains routing metadata
|
||||
for the generation boundary and is not mapped into the provider payload.
|
||||
|
||||
Construction validates the configured base URL and clones any supplied
|
||||
`http.Client` so Promptkit can apply its timeout default without mutating the
|
||||
caller's client. Generation then:
|
||||
@@ -31,6 +36,10 @@ caller's client. Generation then:
|
||||
5. performs the outbound request under the applicable deadlines; and
|
||||
6. decodes the first response choice and token usage.
|
||||
|
||||
`internal/llm` owns the set of reserved OpenAI-compatible request fields used
|
||||
when validating extra parameters. Backend registration consumes the same rule
|
||||
without making the model client depend on registry configuration.
|
||||
|
||||
The implementation has no retry loop, tool-call support, provider catalog,
|
||||
inbound HTTP behavior, or durable session store.
|
||||
|
||||
@@ -51,5 +60,7 @@ The
|
||||
[OpenAI-compatible client tests](../../internal/llm/openai_compatible_client_test.go)
|
||||
own configuration, client cloning, deterministic deadline precedence,
|
||||
authentication, request and response mapping, malformed data, error identity,
|
||||
cancellation, and response-body suppression. They use local test servers and
|
||||
test transports; the default suite makes no live or paid provider requests.
|
||||
cancellation, and response-body suppression. The root transport contract test
|
||||
also verifies that resolved backend settings reach this client without
|
||||
serializing backend identity. All use local test servers or test transports;
|
||||
the default suite makes no live or paid provider requests.
|
||||
|
||||
@@ -11,19 +11,22 @@ contributor workflow and validation.
|
||||
|
||||
| Component | Implemented responsibility | References |
|
||||
| --- | --- | --- |
|
||||
| Root `promptkit` package | Provides the supported engine facade, source and injection options, public request and result values, built-in profile construction, extension interfaces, value conversion, redacted formatting, and public error mapping. | [Package GoDoc](../../doc.go), [engine assembly](../../engine.go) |
|
||||
| Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request and result values, profile construction, extension interfaces, value conversion, redacted formatting, and public error mapping. | [Package GoDoc](../../doc.go), [backend API](../../backends.go), [engine assembly](../../engine.go) |
|
||||
| `examples/go-library/prepare` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, and `Prepare`. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
|
||||
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
|
||||
| `internal/backend` | Constructs each engine's immutable registry from the built-in OpenRouter definition and consumer additions, validates and defensively copies definitions through the shared JSON-value package, and consumes the LLM-owned OpenAI-compatible reserved request-field rule. | [Backend registry](../../internal/backend/registry.go) |
|
||||
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation. | [Domain declarations](../../internal/domain/domain.go) |
|
||||
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
|
||||
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
|
||||
| `internal/jsonvalue` | Validates and deeply copies JSON-compatible extra-parameter trees while preserving supported concrete value types. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
|
||||
| `internal/promptdef` | Loads strictly decoded, validated prompt definitions from filesystem and `fs.FS` sources, including version selection and contained file-backed message content. | [Framework formats](../formats.md), [prompt-definition repository](../../internal/promptdef/filesystem_repository.go) |
|
||||
| `internal/profile` | Loads strictly decoded, validated execution profiles from filesystem and `fs.FS` sources and composes repositories with error-preserving fallback. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go) |
|
||||
| `internal/profile/builtin` | Embeds the built-in execution profile catalog and combines it with an optional primary repository. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) |
|
||||
| `internal/profile` | Loads strictly decoded, validated execution profiles, including backend selection, from filesystem and `fs.FS` sources and composes repositories with error-preserving fallback. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go) |
|
||||
| `internal/profile/builtin` | Embeds the built-in profile catalog, whose entries select OpenRouter, and combines it with an optional primary repository. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) |
|
||||
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
|
||||
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
|
||||
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
|
||||
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests, response decoding, authentication, and deadline handling. | [Internal model client](llm.md) |
|
||||
| `internal/usecase` | Coordinates preparation and execution across internal sources, rendering, artifact loading, generation, validation, and optional repair. | [Internal runner](runner.md) |
|
||||
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
|
||||
| `internal/usecase` | Resolves backend, profile, and request settings and coordinates preparation and execution across internal sources, rendering, artifact loading, generation, validation, and optional repair. | [Internal runner](runner.md) |
|
||||
|
||||
The root package assembles these internal components without exposing their
|
||||
representations. Consumers depend only on the root facade.
|
||||
|
||||
@@ -17,10 +17,12 @@ and override semantics consumed by the runner.
|
||||
## Collaborators
|
||||
|
||||
`Runner` coordinates narrow internal interfaces for prompt definitions,
|
||||
profiles, artifacts, rendering, model generation, and validation. Schema
|
||||
documents are loaded through the validator's optional schema-loader interface.
|
||||
An output repairer can be injected internally, but the ordinary runner
|
||||
constructor does not enable one.
|
||||
profiles, backend resolution, artifacts, rendering, model generation, and
|
||||
validation. The root engine supplies one immutable registry containing the
|
||||
built-in backend and validated consumer additions. Schema documents are loaded
|
||||
through the validator's optional schema-loader interface. An output repairer
|
||||
can be injected internally, but the ordinary runner constructor does not
|
||||
enable one.
|
||||
|
||||
Each invocation carries its state in request, prepared-run, and result values.
|
||||
The runner has no durable run or session store.
|
||||
@@ -29,23 +31,42 @@ The runner has no durable run or session store.
|
||||
|
||||
`Prepare` performs the reusable pre-generation workflow:
|
||||
|
||||
1. validate the prompt selection and load the prompt definition;
|
||||
2. hash the loaded definition;
|
||||
1. validate the required prompt selection and normalize any direct session ID;
|
||||
2. load the prompt definition and hash the original definition;
|
||||
3. select the request profile or the prompt's default profile;
|
||||
4. resolve application-neutral defaults, profile values, and explicit request
|
||||
overrides in that order;
|
||||
5. validate endpoint, model, numeric overrides, and credential requirements;
|
||||
6. resolve the output contract and load a structured-output schema when
|
||||
4. resolve the profile's backend ID, when present;
|
||||
5. resolve application-neutral defaults, backend defaults, profile values,
|
||||
and explicit request overrides in that order;
|
||||
6. validate endpoint, model, numeric overrides, and credential requirements;
|
||||
7. resolve the output contract and load a structured-output schema when
|
||||
required;
|
||||
7. load and hash input artifacts;
|
||||
8. render and hash the prompt; and
|
||||
9. return the effective settings, source identities, messages, hashes, and
|
||||
8. load and hash input artifacts;
|
||||
9. render messages, resolve the effective session ID, and hash the effective
|
||||
rendered prompt; and
|
||||
10. return the effective settings, source identities, messages, hashes, and
|
||||
preparation timing.
|
||||
|
||||
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
|
||||
out-of-range values fail as invalid requests. A direct API key takes
|
||||
precedence over environment lookup for execution; secret values remain
|
||||
excluded from serialized metadata.
|
||||
out-of-range values fail as invalid requests. Endpoint overrides do not change
|
||||
the selected backend identity. Non-empty extra-parameter maps replace whole
|
||||
lower-precedence maps. A direct API key takes precedence over environment
|
||||
lookup; otherwise request, profile, and backend environment-variable names
|
||||
apply in that order. A profile requiring a direct key clears an inherited
|
||||
backend environment name unless the request supplies its own name. Secret
|
||||
values remain excluded from serialized metadata.
|
||||
|
||||
Reasoning overrides are tri-state: nil inherits the profile, a pointer to a
|
||||
nonblank string trims and replaces it, and a pointer to a blank string clears
|
||||
it. A nonblank direct session is normalized before source loading, bypasses
|
||||
the prompt session template, and is applied after ordinary message rendering.
|
||||
A blank direct value retains prompt-template behavior. The runner clears the
|
||||
template only on a value copy of the definition, so the definition hash always
|
||||
describes the original source while the rendered-prompt hash includes the
|
||||
effective direct or rendered session.
|
||||
|
||||
The registry is read-only after engine construction. Concurrent `Prepare` and
|
||||
`Run` calls resolve independent defensive backend values and keep all
|
||||
invocation state local.
|
||||
|
||||
## Run Flow
|
||||
|
||||
@@ -56,12 +77,16 @@ an inability to perform validation is an operational error.
|
||||
|
||||
When an internal repairer is present, a JSON or JSON Schema content failure can
|
||||
trigger bounded repair attempts. Repair receives the effective execution
|
||||
target, validation errors, prior output, and structured-output specification.
|
||||
This capability remains internal and is not a public option.
|
||||
target and session ID, validation errors, prior output, and structured-output
|
||||
specification. This capability remains internal and is not a public option.
|
||||
|
||||
A successful result includes the output artifact and raw output, validation
|
||||
state, prompt and rendered-prompt hashes, selected profile, effective settings,
|
||||
input hashes, token usage, a generated run identifier, and UTC timing.
|
||||
state, effective session ID, prompt and rendered-prompt hashes, selected
|
||||
profile and backend, effective settings, input hashes, token usage, a generated
|
||||
run identifier, and UTC timing. The same effective session reaches initial
|
||||
generation and any repair attempt through the rendered prompt. The same
|
||||
effective target, including backend identity, reaches generation and any
|
||||
repair attempt.
|
||||
|
||||
## Failure Categories
|
||||
|
||||
@@ -71,12 +96,17 @@ validation failures. Wrapping preserves the package identities mapped by the
|
||||
public facade and retains collaborator identities where they are part of the
|
||||
internal contract. Context cancellation propagates through the invoked
|
||||
collaborator and is classified by the owning operation.
|
||||
An overlong direct session is an invalid request before source loading, while
|
||||
an invalid or overlong prompt session template remains a prompt-render failure.
|
||||
An unknown selected backend, or a selected backend with no configured resolver,
|
||||
is classified as a profile-load failure.
|
||||
|
||||
## Test Ownership And Changes
|
||||
|
||||
The [runner tests](../../internal/usecase/runner_test.go) own preparation order,
|
||||
selection and override precedence, schema-before-generation behavior, hashing,
|
||||
generation and validation outcomes, bounded repair, credentials and redaction,
|
||||
selection and override precedence, direct-session resolution,
|
||||
schema-before-generation behavior, hashing, generation and validation
|
||||
outcomes, backend propagation, bounded repair, credentials and redaction,
|
||||
error categories, artifact metadata, usage, and timing.
|
||||
|
||||
Changes to orchestration should continue to use the existing package
|
||||
|
||||
@@ -24,13 +24,20 @@ duplicate detection, and source containment:
|
||||
|
||||
`internal/profile` loads and validates execution profiles from an
|
||||
operating-system filesystem or an `fs.FS`. It supports a primary repository
|
||||
with fallback only when the primary reports that a profile is absent.
|
||||
with fallback only when the primary reports that a profile is absent. Strict
|
||||
YAML decoding recognizes the optional `backend` field, trims its value, and
|
||||
requires a model plus at least one non-blank backend or endpoint. Loading does
|
||||
not check registry membership because the available registry belongs to the
|
||||
assembled engine; the runner checks membership during preparation.
|
||||
|
||||
`internal/profile/builtin` embeds the maintained built-in profile catalog and
|
||||
can place a caller-selected repository ahead of that catalog. Profile behavior
|
||||
is owned by the
|
||||
can place a caller-selected repository ahead of that catalog. Every embedded
|
||||
profile selects `openrouter` and inherits its endpoint and credential
|
||||
environment-variable name from the built-in backend registry rather than
|
||||
repeating those values. Profile behavior is owned by the
|
||||
[profile repository tests](../../internal/profile/repository_test.go), while
|
||||
catalog completeness, duplicate IDs, and overlay behavior are owned by the
|
||||
catalog completeness, the backend-selection invariant, duplicate IDs, and
|
||||
overlay behavior are owned by the
|
||||
[built-in repository tests](../../internal/profile/builtin/repository_test.go).
|
||||
|
||||
## Ordinary Artifacts
|
||||
|
||||
@@ -21,10 +21,14 @@ The implemented internal components consist of:
|
||||
|
||||
- `internal/domain`, which owns framework data values shared by later internal
|
||||
components;
|
||||
- `internal/backend`, which owns validated immutable OpenAI-compatible backend
|
||||
definitions and the built-in OpenRouter definition;
|
||||
- `internal/defaults`, which owns application-neutral framework defaults and
|
||||
constructs the default execution target;
|
||||
- `internal/filecatalog`, which discovers YAML files and provides source-path
|
||||
helpers for filesystem and `fs.FS` consumers;
|
||||
- `internal/jsonvalue`, which validates and defensively copies JSON-compatible
|
||||
extra-parameter trees;
|
||||
- `internal/promptdef`, which loads and validates prompt definitions from
|
||||
filesystem and `fs.FS` sources;
|
||||
- `internal/profile`, which loads, validates, and overlays execution profiles
|
||||
@@ -41,21 +45,23 @@ The implemented internal components consist of:
|
||||
- `internal/usecase`, which coordinates preparation and execution across the
|
||||
internal framework components.
|
||||
|
||||
The `examples/go-library/prepare` package is a maintained downstream consumer
|
||||
of the root facade. It does not expose a library package or participate in
|
||||
internal assembly.
|
||||
The `examples/go-library/prepare` and `examples/go-library/run` packages are
|
||||
maintained downstream consumers of the root facade. They do not expose library
|
||||
packages or participate in internal assembly.
|
||||
|
||||
The root facade assembles the internal repositories, renderer, validator,
|
||||
outbound client, and use-case runner while translating public values and
|
||||
errors at the library boundary. The defaults and renderer depend on the domain
|
||||
model. Prompt-definition and profile repositories use the domain model, file
|
||||
catalog, and YAML decoder. The built-in profile repository supplies an
|
||||
embedded `fs.FS` to the profile package. Artifact reading uses the domain model
|
||||
and application-neutral defaults. Validation uses the domain model, file
|
||||
catalog, and JSON Schema implementation. The model client uses the domain
|
||||
model, application-neutral defaults, and an injected or standard-library HTTP
|
||||
client. The use-case runner depends on the narrow interfaces owned by each
|
||||
internal component.
|
||||
The root facade assembles one immutable backend registry, the internal
|
||||
repositories, renderer, validator, outbound client, and use-case runner while
|
||||
translating public values and errors at the library boundary. The registry
|
||||
contains built-ins plus validated engine-scoped consumer additions. The
|
||||
defaults and renderer depend on the domain model. Prompt-definition and
|
||||
profile repositories use the domain model, file catalog, and YAML decoder. The
|
||||
built-in profile repository supplies an embedded `fs.FS` to the profile
|
||||
package. Artifact reading uses the domain model and application-neutral
|
||||
defaults. Validation uses the domain model, file catalog, and JSON Schema
|
||||
implementation. The model client uses the domain model, application-neutral
|
||||
defaults, and an injected or standard-library HTTP client. The use-case runner
|
||||
depends on the narrow interfaces owned by each internal component, including
|
||||
backend lookup.
|
||||
|
||||
The current implementation follows this dependency direction:
|
||||
|
||||
@@ -72,9 +78,13 @@ downstream consumers, including Scriptorium
|
||||
narrow injected abstractions
|
||||
```
|
||||
|
||||
The facade coordinates internal components and adapts the supported public
|
||||
extension interfaces to narrow internal abstractions. Internal components must
|
||||
not depend on consumers or on Scriptorium.
|
||||
The backend registry depends on the domain model and shared JSON-value
|
||||
validation, has no mutation API after construction, and consumes the
|
||||
OpenAI-compatible reserved request-field rule owned by the model client. The
|
||||
model client does not depend on registry configuration. The facade coordinates
|
||||
internal components and adapts
|
||||
the supported public extension interfaces to narrow internal abstractions.
|
||||
Internal components must not depend on consumers or on Scriptorium.
|
||||
|
||||
## Repository And Consumer Boundary
|
||||
|
||||
|
||||
254
docs/release.md
254
docs/release.md
@@ -7,27 +7,91 @@ tags. It does not publish runnable binaries or binary packages and does not
|
||||
currently use hosted CI. The release maintainer performs and records the
|
||||
required validation.
|
||||
|
||||
The first planned release is `v0.1.0`. Do not create that tag until the
|
||||
framework has been extracted and the resulting public library has passed this
|
||||
procedure. Later tags use the `vMAJOR.MINOR.PATCH` form. While Promptkit remains
|
||||
pre-`v1`, release notes must identify intentional public API changes and any
|
||||
consumer migration required by them.
|
||||
`v0.1.0` is the initial published release. Later releases use semantic
|
||||
`vMAJOR.MINOR.PATCH` tags. Before `v1`, minor releases may change the public
|
||||
API and patch releases preserve compatibility within their minor line. Every
|
||||
pre-`v1` release note must summarize compatibility, identify public API
|
||||
changes, and state any action required of consumers.
|
||||
|
||||
## Prepare The Release
|
||||
Promptkit releases are source-only. The annotated tag message is the release
|
||||
note; there is no separate hosted release or binary packaging step.
|
||||
|
||||
Work from a clean checkout of the intended release commit, outside any Go
|
||||
workspace and without a local module replacement. Confirm the source commit is
|
||||
already published through the normal branch workflow.
|
||||
## Establish The Candidate
|
||||
|
||||
From the Promptkit repository root, verify the checkout:
|
||||
Choose a version that has not been published and export it as
|
||||
`RELEASE_VERSION`. Run every command in this procedure from the Promptkit
|
||||
repository root in the same POSIX shell. Do not reuse `v0.1.0` or another
|
||||
existing version.
|
||||
|
||||
The following guard derives the release commit from `HEAD` and stops on a
|
||||
missing or malformed version, a checkout other than synchronized `main`,
|
||||
uncommitted changes, an active Go workspace, a module replacement, a vendor
|
||||
tree, or an existing local or remote tag:
|
||||
|
||||
```sh
|
||||
gowork=$(go env GOWORK)
|
||||
test -z "$gowork" || test "$gowork" = off
|
||||
test -z "$(git status --short)"
|
||||
git fetch --tags origin
|
||||
set -eu
|
||||
|
||||
: "${RELEASE_VERSION:?export an unpublished vMAJOR.MINOR.PATCH version}"
|
||||
if ! printf '%s\n' "$RELEASE_VERSION" |
|
||||
grep -Eq '^v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$'
|
||||
then
|
||||
printf '%s\n' "invalid release version: $RELEASE_VERSION" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
RELEASE_COMMIT=$(git rev-parse --verify 'HEAD^{commit}')
|
||||
export RELEASE_COMMIT
|
||||
|
||||
check_release_candidate() {
|
||||
test "$(git branch --show-current)" = main
|
||||
test -z "$(git status --porcelain)"
|
||||
|
||||
gowork_value=$(go env GOWORK)
|
||||
case "$gowork_value" in
|
||||
''|off) ;;
|
||||
*)
|
||||
printf '%s\n' "active Go workspace: $gowork_value" >&2
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
|
||||
test -z "$(git ls-files go.work go.work.sum)"
|
||||
test ! -e vendor
|
||||
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod
|
||||
then
|
||||
printf '%s\n' 'go.mod contains a replacement' >&2
|
||||
return 1
|
||||
fi
|
||||
|
||||
git fetch origin main --tags
|
||||
test "$RELEASE_COMMIT" = \
|
||||
"$(git rev-parse --verify 'refs/remotes/origin/main^{commit}')"
|
||||
|
||||
if git show-ref --verify --quiet "refs/tags/$RELEASE_VERSION"
|
||||
then
|
||||
printf '%s\n' "local tag already exists: $RELEASE_VERSION" >&2
|
||||
return 1
|
||||
fi
|
||||
if test -n "$(
|
||||
git ls-remote --tags origin \
|
||||
"refs/tags/$RELEASE_VERSION" \
|
||||
"refs/tags/$RELEASE_VERSION^{}"
|
||||
)"
|
||||
then
|
||||
printf '%s\n' "remote tag already exists: $RELEASE_VERSION" >&2
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
check_release_candidate
|
||||
```
|
||||
|
||||
Do not continue unless the guard completes successfully. In particular, push
|
||||
the intended commit through the normal `main` branch workflow before release;
|
||||
the tag procedure is not a substitute for publishing the source commit.
|
||||
|
||||
## Validate The Candidate
|
||||
|
||||
Confirm the module and root package metadata:
|
||||
|
||||
```sh
|
||||
@@ -42,7 +106,7 @@ gitea.maximumdirect.net/eric/promptkit 1.25.5
|
||||
promptkit gitea.maximumdirect.net/eric/promptkit
|
||||
```
|
||||
|
||||
Run the same default Go validation required by the
|
||||
Run the complete maintainer validation required by the
|
||||
[development guide](development.md):
|
||||
|
||||
```sh
|
||||
@@ -53,84 +117,158 @@ go build ./...
|
||||
go run ./examples/go-library/prepare
|
||||
```
|
||||
|
||||
Check every tracked Go file and repository whitespace:
|
||||
Check every tracked Go file. This command must produce no output:
|
||||
|
||||
```sh
|
||||
gofmt -l $(git ls-files '*.go')
|
||||
unformatted=$(
|
||||
git ls-files '*.go' |
|
||||
while IFS= read -r go_file
|
||||
do
|
||||
gofmt -l "$go_file"
|
||||
done
|
||||
)
|
||||
test -z "$unformatted"
|
||||
```
|
||||
|
||||
Follow every maintained Markdown link and confirm that its local or published
|
||||
target exists. Review the repository for generated binaries, test or coverage
|
||||
output, credentials, template residue, downloaded assets, and other files that
|
||||
do not belong in source control.
|
||||
|
||||
Recheck module and repository hygiene, whitespace, and the clean checkout:
|
||||
|
||||
```sh
|
||||
test -z "$(git ls-files go.work go.work.sum)"
|
||||
test ! -e vendor
|
||||
if grep -Eq '^[[:space:]]*replace([[:space:]]|\()' go.mod
|
||||
then
|
||||
printf '%s\n' 'go.mod contains a replacement' >&2
|
||||
exit 1
|
||||
fi
|
||||
git diff --check
|
||||
test -z "$(git status --porcelain)"
|
||||
```
|
||||
|
||||
The formatting command must produce no paths. Follow every maintained Markdown
|
||||
link and confirm its target exists. Review the repository for generated
|
||||
binaries, test or coverage output, credentials, template residue, and other
|
||||
files that do not belong in source control.
|
||||
## Write The Release Note
|
||||
|
||||
Confirm that no workspace override is tracked and that `go.mod` contains no
|
||||
`replace` directive:
|
||||
Prepare a plain-text annotated-tag message outside the repository and export
|
||||
its path as `RELEASE_NOTES_FILE`. Use this form, replacing each summary with
|
||||
release-specific text; write `None.` when there are no public API changes or
|
||||
consumer actions:
|
||||
|
||||
```sh
|
||||
git ls-files go.work go.work.sum
|
||||
rg -n '^replace\b' go.mod
|
||||
```text
|
||||
Promptkit vMAJOR.MINOR.PATCH
|
||||
|
||||
Validated commit: full commit ID
|
||||
Compatibility: compatibility summary
|
||||
Public API changes: changes or None.
|
||||
Consumer action: required action or None.
|
||||
```
|
||||
|
||||
Both commands must produce no output. Re-run `git status --short` and require a
|
||||
clean result after every validation and review check.
|
||||
|
||||
## Create And Publish The Tag
|
||||
|
||||
Choose the semantic version from the intended compatibility change. Record the
|
||||
release commit before tagging:
|
||||
After writing it, require all release-note fields, the selected version, and
|
||||
the validated commit to be present:
|
||||
|
||||
```sh
|
||||
release_version=v0.1.0
|
||||
release_commit=$(git rev-parse HEAD)
|
||||
: "${RELEASE_NOTES_FILE:?export the path to the release-note file}"
|
||||
test -f "$RELEASE_NOTES_FILE"
|
||||
test -s "$RELEASE_NOTES_FILE"
|
||||
grep -F "Promptkit $RELEASE_VERSION" "$RELEASE_NOTES_FILE"
|
||||
grep -F "Validated commit: $RELEASE_COMMIT" "$RELEASE_NOTES_FILE"
|
||||
grep -F 'Compatibility:' "$RELEASE_NOTES_FILE"
|
||||
grep -F 'Public API changes:' "$RELEASE_NOTES_FILE"
|
||||
grep -F 'Consumer action:' "$RELEASE_NOTES_FILE"
|
||||
```
|
||||
|
||||
Replace the example version for later releases and keep both values in the same
|
||||
shell for the remaining commands. Confirm the tag does not already exist
|
||||
locally or remotely:
|
||||
Inspect the complete message and confirm that it accurately records the
|
||||
compatibility impact, public API changes, and required consumer action.
|
||||
|
||||
## Create And Inspect The Tag
|
||||
|
||||
Run the candidate guard again immediately before tag creation. This ensures
|
||||
that validation or release-note preparation did not change the checkout and
|
||||
that the commit is still published and untagged:
|
||||
|
||||
```sh
|
||||
test -z "$(git tag --list "$release_version")"
|
||||
test -z "$(git ls-remote --tags origin "refs/tags/$release_version")"
|
||||
check_release_candidate
|
||||
```
|
||||
|
||||
Create an annotated tag whose message identifies the release and records that
|
||||
the documented validation passed for the tagged commit:
|
||||
Create the annotated tag from the prepared release note and bind it explicitly
|
||||
to the validated commit:
|
||||
|
||||
```sh
|
||||
git tag --annotate "$release_version" \
|
||||
--message "Promptkit $release_version; documented validation passed for $release_commit"
|
||||
git tag --annotate "$RELEASE_VERSION" \
|
||||
--file "$RELEASE_NOTES_FILE" \
|
||||
"$RELEASE_COMMIT"
|
||||
```
|
||||
|
||||
Inspect the tag before publication:
|
||||
Inspect both the tag message and its source commit before publication:
|
||||
|
||||
```sh
|
||||
git show --no-patch --decorate "$release_version"
|
||||
test "$(git rev-list -n 1 "$release_version")" = "$release_commit"
|
||||
test "$(git cat-file -t "refs/tags/$RELEASE_VERSION")" = tag
|
||||
git show --no-patch --decorate "refs/tags/$RELEASE_VERSION"
|
||||
test "$(
|
||||
git rev-parse --verify "refs/tags/$RELEASE_VERSION^{commit}"
|
||||
)" = "$RELEASE_COMMIT"
|
||||
```
|
||||
|
||||
Publish the tag without relying on a hosting-provider-specific release
|
||||
interface:
|
||||
If inspection finds an error, delete the unpublished local tag, correct the
|
||||
release note or candidate, and repeat the guards. Never move or recreate a tag
|
||||
that has been published.
|
||||
|
||||
## Publish The Selected Tag
|
||||
|
||||
Push only the selected tag ref. Do not use `git push --tags`:
|
||||
|
||||
```sh
|
||||
git push origin "refs/tags/$release_version"
|
||||
git push origin \
|
||||
"refs/tags/$RELEASE_VERSION:refs/tags/$RELEASE_VERSION"
|
||||
```
|
||||
|
||||
## Verify Publication
|
||||
|
||||
Confirm that the remote tag object matches the local annotated tag and still
|
||||
resolves to the intended source commit:
|
||||
Compare the remote annotated-tag object with the local object, then compare the
|
||||
remote peeled source commit with the validated commit:
|
||||
|
||||
```sh
|
||||
remote_tag=$(git ls-remote --tags origin "refs/tags/$release_version" | awk '{print $1}')
|
||||
test "$remote_tag" = "$(git rev-parse "refs/tags/$release_version")"
|
||||
test "$(git rev-list -n 1 "refs/tags/$release_version")" = "$release_commit"
|
||||
remote_tag=$(
|
||||
git ls-remote --tags origin "refs/tags/$RELEASE_VERSION" |
|
||||
awk 'NR == 1 { print $1 }'
|
||||
)
|
||||
remote_commit=$(
|
||||
git ls-remote --tags origin "refs/tags/$RELEASE_VERSION^{}" |
|
||||
awk 'NR == 1 { print $1 }'
|
||||
)
|
||||
test -n "$remote_tag"
|
||||
test "$remote_tag" = \
|
||||
"$(git rev-parse --verify "refs/tags/$RELEASE_VERSION")"
|
||||
test "$remote_commit" = "$RELEASE_COMMIT"
|
||||
```
|
||||
|
||||
Promptkit must publish the required tag before Scriptorium or another consumer
|
||||
publishes a release that depends on that version. Released consumer modules
|
||||
must not use a local replacement or unpublished Promptkit revision.
|
||||
Finally, resolve the version as an ordinary Go module in a temporary module
|
||||
outside this repository and without a workspace or replacement:
|
||||
|
||||
```sh
|
||||
resolution_dir=$(mktemp -d)
|
||||
(
|
||||
trap 'rm -rf "$resolution_dir"' 0 1 2 15
|
||||
cd "$resolution_dir"
|
||||
GOWORK=off go mod init example.com/promptkit-release-check
|
||||
GOWORK=off go mod download \
|
||||
"gitea.maximumdirect.net/eric/promptkit@$RELEASE_VERSION"
|
||||
resolved_version=$(
|
||||
GOWORK=off go list -m -f '{{.Version}}' \
|
||||
"gitea.maximumdirect.net/eric/promptkit@$RELEASE_VERSION"
|
||||
)
|
||||
test "$resolved_version" = "$RELEASE_VERSION"
|
||||
)
|
||||
```
|
||||
|
||||
Promptkit must publish and verify the required version before Scriptorium or
|
||||
another consumer publishes a release that depends on it. This ordering does
|
||||
not replace the consumer project's own release procedure. Released consumers
|
||||
must select the published Promptkit tag through ordinary module resolution,
|
||||
without a workspace, replacement, vendored Promptkit source, or unpublished
|
||||
revision.
|
||||
|
||||
## Policy Changes
|
||||
|
||||
|
||||
236
docs/roadmap/concurrency.md
Normal file
236
docs/roadmap/concurrency.md
Normal file
@@ -0,0 +1,236 @@
|
||||
# Backend-Specific Concurrency Management
|
||||
|
||||
**Status:** Accepted.
|
||||
|
||||
## Purpose
|
||||
|
||||
This roadmap defines the scope and target end state for engine-local,
|
||||
backend-specific concurrency management. It records the intended capability,
|
||||
consumer value, and important policy choices.
|
||||
|
||||
This document is planning material, not a description of current behavior.
|
||||
Current exported contracts remain owned by Go declarations and GoDoc, backend
|
||||
registration guidance by the
|
||||
[consumer guide](../consumers/pkg-promptkit.md#register-a-custom-backend), and
|
||||
implemented orchestration by the
|
||||
[internal runner document](../internal/runner.md).
|
||||
|
||||
## Motivation
|
||||
|
||||
Different model backends can sustain very different request loads. A local
|
||||
network endpoint may need a small concurrency limit, while OpenRouter can
|
||||
usually accept substantially more simultaneous work. Requiring every consumer
|
||||
to build its own semaphores and queues would duplicate routing knowledge,
|
||||
create inconsistent cancellation behavior, and make it easy for one caller to
|
||||
bypass the intended backend limit.
|
||||
|
||||
Promptkit should own this coordination because it already resolves each run to
|
||||
an engine-scoped backend identity and owns every model-generation call made by
|
||||
the runner. Consumers should continue submitting ready-to-run requests through
|
||||
the synchronous API, including concurrently from multiple goroutines, without
|
||||
implementing their own backend scheduler.
|
||||
|
||||
The buffered queue is a safety boundary, not an ordinary throughput
|
||||
restriction. Its primary purpose is to prevent a bug or unintended submission
|
||||
loop from creating an unbounded in-memory backlog.
|
||||
|
||||
## Scope
|
||||
|
||||
The feature will add optional concurrency policy to registered backends and
|
||||
coordinate `Run` calls against independent per-backend capacity pools.
|
||||
|
||||
Each policy has two distinct controls:
|
||||
|
||||
- an active-generation limit, which protects the backend from too many
|
||||
simultaneous model requests; and
|
||||
- a bounded waiting capacity, which protects the process from admitting an
|
||||
unbounded backlog.
|
||||
|
||||
Concurrency policy belongs to a backend registration. It is not a profile
|
||||
model parameter and cannot be overridden per run. Profiles select the policy
|
||||
through their backend ID, while a profile or request endpoint override remains
|
||||
in the selected backend's pool.
|
||||
|
||||
`Prepare` does not call a model and will remain outside concurrency admission.
|
||||
|
||||
## Defaults And Configuration
|
||||
|
||||
The built-in OpenRouter backend will use:
|
||||
|
||||
- an active-generation limit of 16; and
|
||||
- a waiting capacity of 1024.
|
||||
|
||||
The waiting default is intentionally generous. Reaching it should indicate
|
||||
abnormal submission pressure rather than normal application behavior.
|
||||
|
||||
Consumer-registered backends will remain unlimited unless the consumer
|
||||
configures an active-generation limit. When a consumer enables a limit and
|
||||
does not specify waiting capacity, the waiting capacity will default to 1024.
|
||||
Consumers may configure a different bounded capacity, including zero when
|
||||
they want no admitted backlog beyond the active-limit-sized run set.
|
||||
|
||||
The public representation must distinguish an omitted waiting capacity from
|
||||
an explicit zero.
|
||||
|
||||
Endpoint-only profiles have no backend registration from which to obtain
|
||||
policy and will remain unlimited. A future engine-wide or endpoint-keyed
|
||||
policy can be considered separately if consumers demonstrate that need.
|
||||
|
||||
Invalid limits or capacities will fail engine construction as invalid
|
||||
configuration. Policy values will be copied into engine-owned immutable state
|
||||
along with the rest of the backend registration.
|
||||
|
||||
## Admission And Execution Behavior
|
||||
|
||||
`Run` remains a synchronous, wait-for-result operation. Concurrent callers may
|
||||
block inside `Run` while waiting for their selected backend, then receive the
|
||||
ordinary result or error from that invocation.
|
||||
|
||||
For a configured pool, the active-generation limit plus the waiting capacity
|
||||
defines the maximum number of concurrent `Run` invocations that Promptkit will
|
||||
accept for that backend. A waiting capacity of zero therefore accepts no more
|
||||
runs than the active limit. Admission is immediate: a call either reserves one
|
||||
of those bounded slots or receives the capacity error. An accepted run may
|
||||
then wait internally for active-generation capacity.
|
||||
|
||||
For a limited backend, Promptkit will bound the number of accepted runs before
|
||||
expensive artifact loading, prompt rendering, and large defensive copies where
|
||||
practical. Lightweight prompt, profile, and backend resolution may occur first
|
||||
when it is required to identify the correct capacity pool. This pre-admission
|
||||
resolution must not become a second execution-precedence path with behavior
|
||||
that can drift from `Prepare`.
|
||||
|
||||
An accepted run retains its admission until it completes or fails. Every
|
||||
actual model-generation call for that run must separately observe the
|
||||
backend's active-generation limit. This includes:
|
||||
|
||||
- the initial generation;
|
||||
- every output-repair generation; and
|
||||
- calls made through either the built-in or an injected model client.
|
||||
|
||||
Preparation and output validation should not hold an active-generation permit.
|
||||
A repair remains part of its already-admitted run, but reacquires active
|
||||
generation capacity so repairs cannot exceed the backend limit. It must not be
|
||||
rejected merely because new runs filled the waiting queue after its initial
|
||||
generation.
|
||||
|
||||
Within one backend pool, waiting generation calls should be served in FIFO
|
||||
order, subject to canceled calls being removed. Different backend pools make
|
||||
progress independently; a saturated local backend must not consume
|
||||
OpenRouter's active or waiting capacity.
|
||||
|
||||
The feature will not promise ordering across backend pools or completion order
|
||||
among admitted runs.
|
||||
|
||||
## Capacity Failure And Cancellation
|
||||
|
||||
When a backend's bounded waiting capacity is full, a new `Run` call will fail
|
||||
promptly rather than waiting outside the bounded admission system. The public
|
||||
API will expose a recognizable capacity-exhaustion error identity distinct
|
||||
from invalid configuration, invalid requests, and model-client failures.
|
||||
Rejected calls return no partial result and do not invoke the model client.
|
||||
|
||||
Waiting within the admitted backlog or for active-generation capacity must
|
||||
honor the caller's context. Cancellation or deadline expiry will:
|
||||
|
||||
- stop waiting promptly;
|
||||
- release any admission or generation capacity held by that invocation;
|
||||
- preserve the applicable context error identity; and
|
||||
- avoid invoking the model client if cancellation wins before generation
|
||||
starts.
|
||||
|
||||
Capacity must also be released after preparation, generation, validation,
|
||||
repair, or collaborator failure. One failed or canceled run must not reduce
|
||||
the backend's future usable capacity.
|
||||
|
||||
Elapsed `Run` timing will include time spent waiting after the call is
|
||||
accepted. `PreparedRun` timing will continue to describe preparation rather
|
||||
than queue waiting.
|
||||
|
||||
## Engine And Client Boundaries
|
||||
|
||||
All pools and queued state belong to one `Engine`. Separate engines do not
|
||||
share capacity, even when they register the same backend ID or endpoint. The
|
||||
feature introduces no process-global scheduler.
|
||||
|
||||
The engine will apply policy consistently to the built-in model client and an
|
||||
injected `LLMClient`. Consumers calling their own client outside Promptkit are
|
||||
outside this boundary. Injected clients remain responsible for their internal
|
||||
thread safety and cancellation behavior.
|
||||
|
||||
Backend policy is keyed by the resolved backend ID rather than endpoint text.
|
||||
This preserves stable routing when a selected backend's endpoint is overridden
|
||||
and avoids accidentally combining unrelated registrations that happen to use
|
||||
the same URL.
|
||||
|
||||
## Queue Lifetime And Observability
|
||||
|
||||
Admission state is buffered, ephemeral, and in-process. It is not persisted
|
||||
and has no survival guarantee across engine disposal or process termination.
|
||||
Promptkit will not introduce background job ownership or require consumers to
|
||||
start or stop workers.
|
||||
|
||||
The initial feature does not require public queue-depth metrics, callbacks, or
|
||||
inspection APIs. Capacity errors and ordinary call timing provide the
|
||||
consumer-visible behavior. Operational observability can be added later
|
||||
without coupling the scheduling mechanism to an application logging or
|
||||
metrics system.
|
||||
|
||||
## Compatibility
|
||||
|
||||
Consumer-registered backends and endpoint-only profiles remain unlimited
|
||||
unless concurrency is explicitly configured, preserving their existing
|
||||
behavior.
|
||||
|
||||
The built-in OpenRouter backend will change from unlimited concurrency to a
|
||||
limit of 16 with a bounded waiting capacity of 1024. Ordinary synchronous
|
||||
calls remain unchanged, while unusually high concurrent use may now wait or
|
||||
return the capacity error. This behavioral change must be identified in the
|
||||
release notes for the version that publishes it.
|
||||
|
||||
Adding backend policy fields and a public capacity error is otherwise
|
||||
additive. The change will use a pre-`v1` minor release under Promptkit's
|
||||
[release policy](../release.md#release-model).
|
||||
|
||||
## Non-Goals
|
||||
|
||||
This scope does not include:
|
||||
|
||||
- asynchronous job handles, polling, or detached result delivery;
|
||||
- durable or cross-process queues;
|
||||
- persistence or recovery across engine or process shutdown;
|
||||
- priorities, scheduling weights, or consumer-defined fairness classes;
|
||||
- automatic retries, backoff, rate-limit interpretation, or provider quota
|
||||
discovery;
|
||||
- token-per-minute or request-per-minute rate limiting;
|
||||
- dynamic reconfiguration after engine construction;
|
||||
- per-profile or per-run concurrency overrides;
|
||||
- endpoint-keyed pooling for profiles without a backend ID;
|
||||
- process-global coordination across engines;
|
||||
- application worker lifecycle, logging, tracing, or metrics policy; or
|
||||
- changes to prompt, profile, schema, or model-provider wire formats.
|
||||
|
||||
## Target End State
|
||||
|
||||
This roadmap reaches its target end state when:
|
||||
|
||||
- each engine independently coordinates configured backend capacity;
|
||||
- the built-in OpenRouter backend allows 16 active generations and up to 1024
|
||||
waiting runs;
|
||||
- consumer backends can opt into their own active and waiting limits while
|
||||
remaining unlimited by default;
|
||||
- endpoint overrides retain the selected backend's capacity pool and
|
||||
endpoint-only profiles remain unlimited;
|
||||
- synchronous `Run` callers wait for and receive their ordinary result;
|
||||
- admission is bounded before expensive preparation work where practical;
|
||||
- every initial and repair generation observes the backend's active limit
|
||||
without serializing preparation or validation;
|
||||
- a full waiting queue returns a recognizable capacity error without invoking
|
||||
the model client;
|
||||
- cancellation and all failure paths promptly release capacity and preserve
|
||||
context error identity;
|
||||
- built-in and injected model clients receive the same scheduling behavior;
|
||||
- pools remain ephemeral, engine-scoped, and independent across backend IDs;
|
||||
and
|
||||
- current-state GoDoc, consumer, internal, and release documentation describe
|
||||
the implemented behavior once it lands.
|
||||
53
docs/roadmap/future.md
Normal file
53
docs/roadmap/future.md
Normal file
@@ -0,0 +1,53 @@
|
||||
# Future Feature Ideas
|
||||
|
||||
## Purpose
|
||||
|
||||
This document catalogs reasonably specific ideas that may be useful in future
|
||||
Promptkit development. It is an idea pool, not a commitment, schedule, or
|
||||
description of current behavior.
|
||||
|
||||
Ideas belong here while they are worth retaining but have not been selected
|
||||
for active development. Keep each entry at the level of intended capability,
|
||||
consumer value, and important scope boundaries. Defer API design,
|
||||
implementation details, sequencing, and acceptance criteria until an idea is
|
||||
selected.
|
||||
|
||||
## Using This Catalog
|
||||
|
||||
- Add an idea when its purpose and likely value can be stated clearly.
|
||||
- Keep entries independent enough that maintainers can evaluate and select
|
||||
them individually.
|
||||
- Note significant dependencies or boundary concerns, but do not turn entries
|
||||
into implementation plans.
|
||||
- Treat inclusion as an invitation to evaluate, not as approval or priority.
|
||||
- When an idea is selected, move its active planning to a focused roadmap or,
|
||||
when it requires a durable architectural decision, an ADR. Update
|
||||
current-state documentation only when implementation lands.
|
||||
- Remove ideas that are no longer relevant. Retain a rejected idea only when
|
||||
its rationale is likely to prevent repeated reconsideration.
|
||||
|
||||
Future capabilities must continue to respect the
|
||||
[architecture policy](../policy/architecture.md), particularly Promptkit's
|
||||
role as an application-neutral library and its boundary with downstream
|
||||
consumers.
|
||||
|
||||
## Ideas
|
||||
|
||||
No ideas are currently cataloged. Backend-specific concurrency management has
|
||||
been selected for active planning in the
|
||||
[focused concurrency roadmap](concurrency.md).
|
||||
|
||||
## Entry Format
|
||||
|
||||
Use a short heading followed by a concise summary. Add focused bullets when
|
||||
they help preserve important scope boundaries without becoming an
|
||||
implementation plan:
|
||||
|
||||
```markdown
|
||||
### Idea name
|
||||
|
||||
Describe the intended capability, who benefits, and the most important scope
|
||||
boundary or dependency.
|
||||
|
||||
- Optionally record an important behavior or boundary.
|
||||
```
|
||||
828
docs/roadmap/implementation.md
Normal file
828
docs/roadmap/implementation.md
Normal file
@@ -0,0 +1,828 @@
|
||||
# Backend-Specific Concurrency Management Implementation Plan
|
||||
|
||||
**Status:** Ready for implementation.
|
||||
|
||||
## Purpose
|
||||
|
||||
This document is the decision-complete implementation plan for
|
||||
[backend-specific concurrency management](concurrency.md). It is written for a
|
||||
coding agent that will implement each stage in order.
|
||||
|
||||
The feature roadmap owns the intended capability, consumer value, policy
|
||||
choices, compatibility decision, and target end state. This document owns the
|
||||
concrete API, internal representation, scheduling architecture, implementation
|
||||
sequence, test ownership, documentation updates, and completion gates.
|
||||
|
||||
## Implementation Rules
|
||||
|
||||
- Complete the stages in order. Keep the repository compiling and the focused
|
||||
tests passing at every stage boundary.
|
||||
- Preserve unrelated working-tree changes. In particular, `concurrency.md` and
|
||||
the removal of its source idea from `future.md` may already be uncommitted
|
||||
when implementation begins; retain both.
|
||||
- Follow every policy under `docs/policy/`, the task-specific reading guide in
|
||||
`docs/development.md`, and the target behavior in `concurrency.md`.
|
||||
- Keep the public API in the root `promptkit` package and implementation
|
||||
details under `internal/`. Do not expose scheduler types or create another
|
||||
public package.
|
||||
- Use only the Go standard library for scheduling. Do not add a queue,
|
||||
semaphore, worker-pool, or metrics dependency.
|
||||
- Preserve synchronous, wait-for-result `Run`, unrestricted `Prepare`,
|
||||
engine-local state, endpoint-only profiles, backend-selected profiles,
|
||||
backend identity through endpoint overrides, and injected `LLMClient`
|
||||
behavior.
|
||||
- Do not broaden the work into asynchronous jobs, durable queues, retries,
|
||||
rate limiting, dynamic configuration, priorities, worker lifecycle,
|
||||
endpoint-keyed pools, or public queue observability.
|
||||
- Keep all tests deterministic, bounded, offline, and race-safe. Coordinate
|
||||
concurrent tests with channels and barriers rather than timing assumptions
|
||||
or live providers.
|
||||
- Update exact GoDoc with each exported declaration change. Update durable
|
||||
current-state documents only after the corresponding behavior is
|
||||
implemented.
|
||||
- Test configurable mechanisms with small test-owned limits. Assert the exact
|
||||
OpenRouter `16` and default queue `1024` values only at the registry contract
|
||||
that owns those operational defaults.
|
||||
- Do not create a release, change a module version, or tag a commit. The final
|
||||
implementation handoff must identify the built-in OpenRouter behavior change
|
||||
for the next pre-`v1` minor release.
|
||||
|
||||
## Fixed Design
|
||||
|
||||
### Public Backend Configuration
|
||||
|
||||
Append these fields to the existing root `Backend` type in `backends.go`:
|
||||
|
||||
```go
|
||||
type Backend struct {
|
||||
// Existing fields remain unchanged and in their current order.
|
||||
|
||||
ConcurrencyLimit int
|
||||
QueueCapacity *int
|
||||
}
|
||||
```
|
||||
|
||||
Use these exact semantics:
|
||||
|
||||
| Public values | Meaning |
|
||||
| --- | --- |
|
||||
| `ConcurrencyLimit == 0`, `QueueCapacity == nil` | Unlimited backend; preserve current behavior. |
|
||||
| `ConcurrencyLimit > 0`, `QueueCapacity == nil` | Limit active generations and use the default waiting capacity of 1024. |
|
||||
| `ConcurrencyLimit > 0`, `QueueCapacity != nil` | Limit active generations and use the pointed-to capacity exactly, including zero. |
|
||||
| `ConcurrencyLimit < 0` | Invalid engine configuration. |
|
||||
| `QueueCapacity != nil` and `*QueueCapacity < 0` | Invalid engine configuration. |
|
||||
| `ConcurrencyLimit == 0` and `QueueCapacity != nil` | Invalid engine configuration because a queue without an active limit has no defined consumer value. |
|
||||
|
||||
`ConcurrencyLimit` counts simultaneous calls to the engine-owned internal
|
||||
model-client boundary for this backend. `QueueCapacity` controls additional
|
||||
accepted `Run` invocations beyond that limit. The maximum admitted runs for a
|
||||
limited backend is therefore:
|
||||
|
||||
```text
|
||||
ConcurrencyLimit + effective QueueCapacity
|
||||
```
|
||||
|
||||
Guard that addition against integer overflow during backend validation.
|
||||
Do not impose an arbitrary upper bound beyond non-negativity and overflow
|
||||
safety.
|
||||
|
||||
The `QueueCapacity` pointer exists only to distinguish omission from explicit
|
||||
zero. `WithBackend` and `NewEngine` must not retain the caller's pointer.
|
||||
`Backend` continues to have no stable JSON representation, and consumers
|
||||
remain directed to keyed literals.
|
||||
|
||||
Do not add concurrency fields to `Profile`, `ExecutionTarget`,
|
||||
`ExecutionTargetOverride`, `RunRequest`, prompt or profile files, or stable
|
||||
prepared/result JSON.
|
||||
|
||||
### Built-In And Custom Defaults
|
||||
|
||||
The backend registry owns these exact operational defaults:
|
||||
|
||||
```go
|
||||
const (
|
||||
openRouterConcurrencyLimit = 16
|
||||
defaultQueueCapacity = 1024
|
||||
)
|
||||
```
|
||||
|
||||
The built-in `openrouter` definition has a normalized concurrency limit of 16
|
||||
and queue capacity of 1024.
|
||||
|
||||
Consumer registrations remain unlimited when concurrency is omitted. For a
|
||||
consumer backend with a positive limit and omitted queue capacity, normalize
|
||||
the queue capacity to 1024. Preserve an explicitly configured zero.
|
||||
|
||||
Consumers still cannot replace the reserved `openrouter` registration.
|
||||
Endpoint-only profiles have no backend policy and remain unlimited. A selected
|
||||
backend retains its pool when a profile or request overrides only its endpoint.
|
||||
|
||||
### Internal Backend Representation
|
||||
|
||||
Extend `internal/domain.Backend` with scalar policy values and explicit
|
||||
presence rather than retaining a pointer:
|
||||
|
||||
```go
|
||||
type Backend struct {
|
||||
// Existing fields...
|
||||
ConcurrencyLimit int
|
||||
QueueCapacity int
|
||||
QueueCapacitySet bool
|
||||
}
|
||||
|
||||
type BackendCapacityPolicy struct {
|
||||
ConcurrencyLimit int
|
||||
QueueCapacity int
|
||||
}
|
||||
```
|
||||
|
||||
`WithBackend` converts the public pointer into `QueueCapacity` plus
|
||||
`QueueCapacitySet`. Registry normalization validates the combinations above,
|
||||
fills the default, and leaves every limited stored backend with
|
||||
`QueueCapacitySet == true`. Unlimited stored backends retain zero values and
|
||||
`QueueCapacitySet == false`.
|
||||
|
||||
Add this internal registry method:
|
||||
|
||||
```go
|
||||
func (r *Registry) CapacityPolicies() map[string]domain.BackendCapacityPolicy
|
||||
```
|
||||
|
||||
It returns a newly allocated map containing only limited backends. Values are
|
||||
scalars, so callers cannot mutate registry state. The built-in OpenRouter
|
||||
policy is included. `GetBackend` continues returning a defensive backend copy,
|
||||
now including normalized scalar capacity metadata.
|
||||
|
||||
Capacity policy is operational registry metadata. Do not merge it into an
|
||||
execution target or expose it to injected model clients.
|
||||
|
||||
### Public Capacity Error
|
||||
|
||||
Add this root sentinel beside the other run errors in `engine.go`:
|
||||
|
||||
```go
|
||||
var ErrCapacityExceeded = errors.New("backend capacity exceeded")
|
||||
```
|
||||
|
||||
Its GoDoc must state that it identifies a `Run` rejected because the selected
|
||||
backend has already admitted `ConcurrencyLimit + QueueCapacity` runs. It is
|
||||
not an invalid request, an LLM/provider rate-limit response, or an
|
||||
`ErrLLMGenerate` failure.
|
||||
|
||||
The internal capacity component owns a corresponding internal
|
||||
`ErrCapacityExceeded`. Add its mapping in `publicErrorFor` before the broader
|
||||
generation and invalid-request cases. The public error must preserve the
|
||||
internal error through wrapping while matching `ErrCapacityExceeded` with
|
||||
`errors.Is`.
|
||||
|
||||
A capacity rejection returns no partial result and must not invoke the
|
||||
artifact reader, renderer, schema loader, validator, or model client. Prompt,
|
||||
profile, and backend loading needed to select the pool may already have
|
||||
occurred.
|
||||
|
||||
### Internal Capacity Component
|
||||
|
||||
Add `internal/capacity` as the single owner of engine-local run admission and
|
||||
active-generation permits.
|
||||
|
||||
Use these package-level boundaries:
|
||||
|
||||
```go
|
||||
var ErrCapacityExceeded error
|
||||
|
||||
type Manager struct {
|
||||
// Private immutable pool map.
|
||||
}
|
||||
|
||||
func NewManager(
|
||||
policies map[string]domain.BackendCapacityPolicy,
|
||||
) (*Manager, error)
|
||||
|
||||
func (m *Manager) Admit(
|
||||
ctx context.Context,
|
||||
backendID string,
|
||||
) (release func(), err error)
|
||||
|
||||
func NewClient(m *Manager, next llm.Client) llm.Client
|
||||
```
|
||||
|
||||
`NewManager` copies the supplied map and creates one independent pool per
|
||||
limited backend. Defensively reject blank IDs, non-positive concurrency
|
||||
limits, negative queue capacities, or total-capacity overflow even though the
|
||||
registry normally supplies normalized values. Construction creates no worker
|
||||
goroutines.
|
||||
|
||||
An absent manager, blank backend ID, or ID absent from the policy map is
|
||||
unlimited:
|
||||
|
||||
- `Admit` succeeds with a non-nil no-op release function; and
|
||||
- the client wrapper calls the next client directly.
|
||||
|
||||
For a limited pool, `Admit` is immediate and context-aware:
|
||||
|
||||
1. return `ctx.Err()` if the context is already done;
|
||||
2. under the pool lock, compare admitted runs with
|
||||
`ConcurrencyLimit + QueueCapacity`;
|
||||
3. return an error matching internal `ErrCapacityExceeded` when full; or
|
||||
4. increment admitted runs and return an idempotent release function.
|
||||
|
||||
The release function decrements admission exactly once, even if accidentally
|
||||
called more than once. It does not release an active-generation permit; those
|
||||
permits have their own lifetime.
|
||||
|
||||
### FIFO Generation Permits
|
||||
|
||||
`NewClient` returns an internal `llm.Client` wrapper around either the built-in
|
||||
client or the public-client adapter. It must preserve requests, successful
|
||||
responses, nil responses, and collaborator error identities exactly.
|
||||
`next` must be non-nil; `NewEngine` and internal runner construction maintain
|
||||
that invariant. A nil manager returns `next` unchanged.
|
||||
|
||||
For a configured backend ID, the wrapper:
|
||||
|
||||
1. acquires one active-generation permit from the matching pool;
|
||||
2. waits in FIFO order when the active count equals `ConcurrencyLimit`;
|
||||
3. removes a canceled waiter and returns `ctx.Err()` when cancellation wins
|
||||
before the permit is granted;
|
||||
4. invokes the next client only after a permit is granted; and
|
||||
5. releases the permit with `defer` after every success, nil response,
|
||||
collaborator error, panic unwinding, or context outcome.
|
||||
|
||||
Implement FIFO and cancellation explicitly with a mutex and an ordered waiter
|
||||
list. A channel used only as a counting semaphore is insufficient because it
|
||||
does not define FIFO ordering or safe removal of canceled waiters.
|
||||
|
||||
Permit grant and cancellation must have one lock-protected linearization
|
||||
point. If cancellation removes the waiter first, do not invoke the next
|
||||
client. If grant wins first, invoke the next client with the caller's context;
|
||||
the next client may then observe cancellation normally. Never lose or
|
||||
double-release a permit in this race.
|
||||
|
||||
Releasing a permit transfers it to the oldest non-canceled waiter before
|
||||
making it generally available. Different backend pools never share admission
|
||||
or active counts.
|
||||
|
||||
The active wrapper enforces its limit even if an internal caller invokes it
|
||||
without a run admission lease. Bounded backlog is guaranteed for ordinary
|
||||
engine `Run` calls by the runner admission path; no public API exposes the
|
||||
wrapped internal client directly.
|
||||
|
||||
### Engine Assembly
|
||||
|
||||
In `NewEngine`, after constructing the validated backend registry:
|
||||
|
||||
1. obtain `backendRegistry.CapacityPolicies()`;
|
||||
2. construct one `capacity.Manager`;
|
||||
3. construct the selected base internal LLM client exactly as today;
|
||||
4. wrap that base client with `capacity.NewClient`; and
|
||||
5. pass both the wrapped client and manager-as-admitter to the runner.
|
||||
|
||||
Every `NewEngine` call constructs a distinct manager. Do not cache managers,
|
||||
pools, or policies in package globals. The wrapper must be applied after a
|
||||
public injected client is adapted to `internal/llm.Client`, so built-in and
|
||||
injected clients receive identical scheduling behavior.
|
||||
|
||||
If `NewManager` reports a defensive configuration error, make `NewEngine`
|
||||
return an error matching `ErrInvalidConfig`.
|
||||
|
||||
`Prepare` does not use the manager. An injected client remains required to be
|
||||
safe for concurrent calls because different backend pools and unlimited
|
||||
backends may still invoke it concurrently.
|
||||
|
||||
### Shared Two-Phase Preparation
|
||||
|
||||
Refactor `internal/usecase.Runner` so `Prepare` and `Run` share one preparation
|
||||
pipeline with two private phases. Do not duplicate prompt/profile/backend
|
||||
selection or execution precedence.
|
||||
|
||||
The first phase resolves only the state required before admission:
|
||||
|
||||
1. validate `PromptID`;
|
||||
2. normalize the direct session ID;
|
||||
3. load the prompt definition;
|
||||
4. hash the original prompt definition at its existing error-order position;
|
||||
5. select and load the execution profile;
|
||||
6. resolve the selected backend;
|
||||
7. resolve and validate the effective execution target and credentials; and
|
||||
8. resolve the effective output contract without loading its schema.
|
||||
|
||||
Return a private state value containing the loaded definition, normalized
|
||||
direct session, prompt-definition hash, selected profile ID, effective target,
|
||||
numeric-presence metadata, effective output contract, and preparation start
|
||||
time. Keep this value private to `internal/usecase`.
|
||||
|
||||
The second phase consumes that state and performs:
|
||||
|
||||
1. structured-output schema loading;
|
||||
2. artifact loading and input hashing;
|
||||
3. message and prompt-session rendering;
|
||||
4. direct-session application;
|
||||
5. rendered-prompt hashing; and
|
||||
6. `PreparedRun` construction and timing.
|
||||
|
||||
Preserve every existing precedence rule, error identity, direct-session
|
||||
template bypass, hash input, selected identity, copy guarantee, and timing
|
||||
field. Do not reload the prompt, profile, or backend between phases.
|
||||
|
||||
`Runner.Prepare` records its start time, runs both phases consecutively, and
|
||||
never calls admission. Its behavior and error ordering remain unchanged.
|
||||
|
||||
`Runner.Run` records its existing run start time, runs the first preparation
|
||||
phase, and then calls:
|
||||
|
||||
```go
|
||||
release, err := r.admitter.Admit(ctx, effectiveBackendID)
|
||||
```
|
||||
|
||||
Use a narrow use-case-owned interface with the same signature:
|
||||
|
||||
```go
|
||||
type RunAdmitter interface {
|
||||
Admit(context.Context, string) (func(), error)
|
||||
}
|
||||
```
|
||||
|
||||
A nil admitter means unlimited behavior for internal constructors and tests.
|
||||
On successful admission, immediately `defer release()` around the remainder of
|
||||
the run. Then run the second preparation phase, initial generation,
|
||||
validation, and all repair attempts.
|
||||
|
||||
If admission returns internal `capacity.ErrCapacityExceeded`, add useful
|
||||
backend context without changing its identity. If it returns `ctx.Err()`,
|
||||
preserve that identity directly rather than recategorizing it as invalid
|
||||
request or generation failure.
|
||||
|
||||
This refactor intentionally replaces the current literal `Run`-calls-`Prepare`
|
||||
implementation with shared private phases. Update current-state documentation
|
||||
to describe one shared pipeline rather than retaining an inaccurate call-graph
|
||||
claim.
|
||||
|
||||
### Generation And Repair Lifetime
|
||||
|
||||
The admission lease covers the entire accepted run:
|
||||
|
||||
- second-phase preparation;
|
||||
- initial generation;
|
||||
- validation;
|
||||
- every repair; and
|
||||
- all failure and cancellation exits.
|
||||
|
||||
Preparation and validation do not hold an active-generation permit. The
|
||||
wrapped client acquires a permit only around each actual `Generate` call.
|
||||
|
||||
The runner's initial generation already carries the effective backend ID in
|
||||
`GenerateRequest.Target`. Preserve that value. `RepairRequest.Target` and the
|
||||
default repairer's generated request must continue carrying the same backend
|
||||
ID, allowing each repair to reacquire the same pool's active permit.
|
||||
|
||||
When testing or constructing `NewRunnerWithRepairer`, pass the same wrapped
|
||||
client to both the runner and `NewDefaultOutputRepairer`. Do not add capacity
|
||||
state to `RepairRequest`, `ExecutionTarget`, or public generation values.
|
||||
|
||||
A repair remains within its existing admission lease. It waits for a FIFO
|
||||
active permit but never performs a second bounded admission and therefore
|
||||
cannot fail merely because later runs filled the admission capacity.
|
||||
|
||||
### Error And Cancellation Semantics
|
||||
|
||||
The required public outcomes are:
|
||||
|
||||
| Situation | Required error identity |
|
||||
| --- | --- |
|
||||
| Admission capacity is full | `ErrCapacityExceeded` only; not `ErrInvalidRequest` or `ErrLLMGenerate`. |
|
||||
| Context is done before admission succeeds | Preserve `ctx.Err()`; do not return capacity exhaustion. |
|
||||
| Context cancels while waiting for an active permit | Preserve `ctx.Err()` through the existing `ErrLLMGenerate` generation category. |
|
||||
| Wrapped client fails after permit acquisition | Preserve existing `ErrLLMGenerate` and collaborator identities. |
|
||||
| Preparation or validation fails after admission | Preserve its existing category and release admission. |
|
||||
|
||||
Maintain the existing rule that `Run` returns no partial result on any
|
||||
operational error. Do not add queue status to errors or results.
|
||||
|
||||
`RunResult.Duration` continues to start at runner entry and therefore includes
|
||||
pre-admission resolution, accepted preparation, and active-permit waiting.
|
||||
`PreparedRun.DurationMS` continues to cover only its shared preparation phases;
|
||||
it does not include later generation waiting. Capacity-rejected calls have no
|
||||
result or timing value.
|
||||
|
||||
### Ownership And Concurrency Safety
|
||||
|
||||
The registry, capacity policy map, pool map, and per-pool limits are immutable
|
||||
after engine construction. Only admission counts, active counts, and waiter
|
||||
lists are mutable and must be protected by the owning pool mutex.
|
||||
|
||||
Do not retain public queue pointers, caller request values, contexts, or
|
||||
generation requests after their call completes. A canceled waiter must be
|
||||
unlinked so its context and request cannot remain reachable from the pool.
|
||||
|
||||
Do not hold a pool mutex while:
|
||||
|
||||
- loading or rendering prompts;
|
||||
- reading artifacts or schemas;
|
||||
- invoking a model client;
|
||||
- validating output;
|
||||
- closing a waiter notification channel if the implementation could re-enter
|
||||
pool code; or
|
||||
- calling consumer code.
|
||||
|
||||
No scheduler operation may spawn a goroutine whose lifetime outlasts the
|
||||
calling `Run`. The zero steady-state goroutine count is part of the
|
||||
in-process/no-worker-lifecycle design.
|
||||
|
||||
## Test Ownership
|
||||
|
||||
Use this ownership split and avoid repeating the full policy matrix at every
|
||||
layer:
|
||||
|
||||
- `internal/backend/registry_test.go` owns normalization, validation, the exact
|
||||
OpenRouter policy, the custom default queue, explicit zero, unlimited
|
||||
omission, and policy-map copying.
|
||||
- `internal/capacity/manager_test.go` owns admission bounds, idempotent release,
|
||||
FIFO active permits, cancellation races, capacity recovery, independent
|
||||
pools, unlimited IDs, and observed peak concurrency.
|
||||
- `internal/capacity/client_test.go` owns wrapper request/response/error
|
||||
transparency and the rule that cancellation before grant does not invoke the
|
||||
next client. Combine these with manager tests if one coherent package test
|
||||
expresses the behavior more clearly.
|
||||
- `internal/usecase/runner_test.go` owns two-phase preparation parity, pool
|
||||
selection, admission before expensive work, admission release across run
|
||||
exits, `Prepare` bypass, and repair reuse of the admitted backend.
|
||||
- Root external-package tests own public configuration conversion, assembled
|
||||
engine-local behavior, endpoint-override routing, injected-client limiting,
|
||||
and public capacity/context error identities.
|
||||
- Existing model-client HTTP tests remain unchanged because scheduling does
|
||||
not alter the OpenAI-compatible wire contract.
|
||||
|
||||
Concurrency tests must use test-owned limits such as one or two and
|
||||
channel-controlled blocking clients. Record observed active and peak counts
|
||||
under a mutex or atomics. Do not use `time.Sleep` to infer queue state.
|
||||
Package-internal tests may inspect a waiter list under its mutex through a
|
||||
small test helper when necessary to establish deterministic FIFO ordering; do
|
||||
not add production metrics or hooks solely for tests.
|
||||
|
||||
Do not add separate tests for trivial scalar copies when registry or assembled
|
||||
behavior already protects them.
|
||||
|
||||
## Stage 1 — Backend Policy And Public Configuration
|
||||
|
||||
**Status:** Pending.
|
||||
|
||||
### Goal
|
||||
|
||||
Add the public and internal backend policy representation, normalize all
|
||||
configured states, and expose immutable normalized policies without changing
|
||||
runtime scheduling yet.
|
||||
|
||||
### Work
|
||||
|
||||
1. Add `ConcurrencyLimit` and `QueueCapacity` to `Backend` in `backends.go`
|
||||
with exact GoDoc for unlimited, defaulted, explicit-zero, invalid, and
|
||||
engine-scoped behavior.
|
||||
2. Convert the public queue pointer into scalar value plus presence in
|
||||
`WithBackend`; do not retain the pointer.
|
||||
3. Add the internal backend policy fields and
|
||||
`BackendCapacityPolicy` to `internal/domain/domain.go`.
|
||||
4. Add the two registry-owned constants and configure the built-in OpenRouter
|
||||
definition with 16 and 1024.
|
||||
5. Extend `normalizeBackend` with the fixed validation, defaulting, explicit
|
||||
zero, and overflow rules.
|
||||
6. Add `Registry.CapacityPolicies`, returning only limited policies in a fresh
|
||||
map.
|
||||
7. Update existing backend composite literals and assertions only where the
|
||||
new fields are relevant. Continue using keyed literals.
|
||||
|
||||
### Tests
|
||||
|
||||
1. Extend the exact built-in registry test with the OpenRouter limit and queue.
|
||||
2. Add one coherent table covering unlimited omission, default queue,
|
||||
explicit-zero queue, negative values, queue-without-limit, and total
|
||||
overflow.
|
||||
3. Extend the registry copy/normalization test to prove returned policy maps
|
||||
cannot mutate registry state.
|
||||
4. Add root coverage only if needed to prove the public pointer/presence
|
||||
conversion; do not reproduce registry validation cases at the facade.
|
||||
|
||||
### Focused Validation
|
||||
|
||||
Run:
|
||||
|
||||
```sh
|
||||
gofmt -w backends.go internal/domain/domain.go \
|
||||
internal/backend/registry.go internal/backend/registry_test.go
|
||||
go test . ./internal/backend
|
||||
go vet . ./internal/backend
|
||||
git diff --check
|
||||
```
|
||||
|
||||
Include another touched Go test file in `gofmt` only if it actually changed.
|
||||
|
||||
### Completion Gate
|
||||
|
||||
This stage is complete when every public configuration state has one normalized
|
||||
internal meaning, OpenRouter exposes exactly 16/1024, custom backends remain
|
||||
unlimited by omission, and no runtime call is scheduled yet.
|
||||
|
||||
## Stage 2 — Engine-Local Capacity Manager
|
||||
|
||||
**Status:** Pending.
|
||||
|
||||
### Goal
|
||||
|
||||
Implement and prove the bounded admission mechanism and FIFO active-generation
|
||||
client wrapper independently of runner orchestration.
|
||||
|
||||
### Work
|
||||
|
||||
1. Add `internal/capacity/manager.go` with the manager, immutable policy copy,
|
||||
per-backend pools, internal error, immediate admission, idempotent release,
|
||||
and FIFO context-aware active permits.
|
||||
2. Add `internal/capacity/client.go` with the transparent `llm.Client` wrapper.
|
||||
3. Use mutex-protected waiter state and an ordered list; explicitly resolve
|
||||
grant-versus-cancel races.
|
||||
4. Ensure unlimited and independent-pool fast paths avoid queue allocation.
|
||||
5. Do not start workers, timers, cleanup goroutines, or process-global state.
|
||||
|
||||
### Tests
|
||||
|
||||
1. Add a compact constructor-validation table for blank IDs, non-positive
|
||||
limits, negative queues, and total-capacity overflow.
|
||||
2. With a small configured policy, prove that exactly
|
||||
`limit + queueCapacity` admissions succeed, the next matches
|
||||
`ErrCapacityExceeded`, and a release permits another admission.
|
||||
3. Prove release is idempotent.
|
||||
4. Drive more blocked client calls than the active limit and assert observed
|
||||
peak concurrency never exceeds that limit.
|
||||
5. Prove FIFO order with deterministic queue-entry synchronization.
|
||||
6. Cancel the first and a middle waiter and prove they are removed, never call
|
||||
the wrapped client, and do not block later waiters.
|
||||
7. Exercise the grant/cancel race repeatedly under `go test -race`, asserting
|
||||
no permit leak or double invocation.
|
||||
8. Prove different backend IDs proceed independently and blank, unknown, or
|
||||
nil-manager paths remain unlimited.
|
||||
9. Prove request values, successful and nil responses, and collaborator errors
|
||||
pass through unchanged after permit acquisition.
|
||||
|
||||
### Focused Validation
|
||||
|
||||
Run:
|
||||
|
||||
```sh
|
||||
gofmt -w internal/capacity/manager.go \
|
||||
internal/capacity/manager_test.go \
|
||||
internal/capacity/client.go \
|
||||
internal/capacity/client_test.go
|
||||
go test ./internal/capacity
|
||||
go test -race ./internal/capacity
|
||||
go vet ./internal/capacity
|
||||
git diff --check
|
||||
```
|
||||
|
||||
If tests are combined into one file, omit the nonexistent file from `gofmt`.
|
||||
|
||||
### Completion Gate
|
||||
|
||||
This stage is complete when the standalone component enforces relational
|
||||
admission and active limits, FIFO cancellation is race-safe, separate pools
|
||||
are independent, and the wrapper is transparent apart from waiting.
|
||||
|
||||
## Stage 3 — Shared Preparation And Early Run Admission
|
||||
|
||||
**Status:** Pending.
|
||||
|
||||
### Goal
|
||||
|
||||
Refactor runner preparation into one shared two-phase pipeline and place
|
||||
bounded admission after backend resolution but before schema, artifact, and
|
||||
rendering work.
|
||||
|
||||
### Work
|
||||
|
||||
1. Add the private pre-admission preparation state and split the existing
|
||||
`Prepare` logic according to the fixed design.
|
||||
2. Make `Runner.Prepare` call both phases without an admitter.
|
||||
3. Add the `RunAdmitter` interface and runner field.
|
||||
4. Update `NewRunner` and `NewRunnerWithRepairer` to accept the optional
|
||||
admitter; update internal call sites with nil until root assembly is wired.
|
||||
5. Change `Runner.Run` to use the first phase, admit by effective backend ID,
|
||||
defer the returned release, and then use the second phase.
|
||||
6. Preserve all existing error precedence, target resolution, hashes,
|
||||
metadata, session behavior, and timing.
|
||||
7. Return capacity and context errors with the fixed identities. Do not invoke
|
||||
later collaborators after rejection.
|
||||
|
||||
### Tests
|
||||
|
||||
1. Keep the existing `Run`/`Prepare` parity coverage passing to prove the
|
||||
shared phases do not drift.
|
||||
2. Add a fake admitter that records backend IDs and release calls.
|
||||
3. Prove a backend-selected run admits with the selected ID even when the
|
||||
endpoint is overridden.
|
||||
4. Prove an endpoint-only run uses the unlimited/blank identity and that
|
||||
`Prepare` never calls admission.
|
||||
5. Reject admission and assert schema, artifact, renderer, validator, repairer,
|
||||
and LLM collaborators are not invoked.
|
||||
6. Prove admission is released after one successful run and representative
|
||||
second-phase, generation, and validation errors. Prefer a small table around
|
||||
the single `defer` invariant rather than duplicating every error test.
|
||||
7. Retain direct-session, backend precedence, credential, hashing, and repair
|
||||
tests unchanged except for constructor arguments.
|
||||
|
||||
### Focused Validation
|
||||
|
||||
Run:
|
||||
|
||||
```sh
|
||||
gofmt -w internal/usecase/runner.go \
|
||||
internal/usecase/runner_test.go
|
||||
go test ./internal/usecase
|
||||
go test -race ./internal/usecase
|
||||
go vet ./internal/usecase
|
||||
git diff --check
|
||||
```
|
||||
|
||||
### Completion Gate
|
||||
|
||||
This stage is complete when `Prepare` remains unrestricted, `Run` admits after
|
||||
one canonical routing phase and before expensive completion work, every exit
|
||||
releases admission, and existing preparation semantics remain unchanged.
|
||||
|
||||
## Stage 4 — Engine Assembly And Public Runtime Contract
|
||||
|
||||
**Status:** Pending.
|
||||
|
||||
### Goal
|
||||
|
||||
Wire one manager into each engine, schedule built-in and injected clients,
|
||||
expose the capacity error, and prove assembled runtime behavior.
|
||||
|
||||
### Work
|
||||
|
||||
1. Add public `ErrCapacityExceeded` and its exact GoDoc in `engine.go`.
|
||||
2. Map internal capacity exhaustion in `errors.go`.
|
||||
3. Construct the manager from the registry policy snapshot in `NewEngine`.
|
||||
4. Wrap the selected internal client after built-in or injected-client
|
||||
selection and pass the manager and wrapped client to the runner.
|
||||
5. Update `Engine`, `NewEngine`, `Run`, `WithLLMClient`, and `LLMClient` GoDoc
|
||||
only where concurrency, capacity, or cancellation statements change.
|
||||
6. Ensure manager-construction errors match `ErrInvalidConfig`.
|
||||
7. For internal repair coverage, construct the default repairer with the same
|
||||
wrapped client used by its runner and confirm repair target backend identity
|
||||
remains intact.
|
||||
|
||||
### Tests
|
||||
|
||||
1. Add an external-package assembled test with a small custom limit and a
|
||||
blocking injected client; assert peak generation equals or remains below
|
||||
the configured limit.
|
||||
2. With queue capacity zero, block one accepted run before generation and
|
||||
assert the next matching-backend run returns `ErrCapacityExceeded`, does not
|
||||
match `ErrInvalidRequest` or `ErrLLMGenerate`, returns no result, and never
|
||||
reaches expensive collaborators or the client.
|
||||
3. In the same or another focused workflow, prove an endpoint override remains
|
||||
in the selected backend's pool.
|
||||
4. Prove two engines with the same backend ID have independent capacity.
|
||||
5. Prove an unlimited custom backend and an endpoint-only profile preserve
|
||||
concurrent behavior.
|
||||
6. Cancel a call waiting for an active permit; assert it matches both
|
||||
`context.Canceled` and `ErrLLMGenerate`, never invokes the injected client,
|
||||
and leaves capacity reusable.
|
||||
7. Add one internal repair workflow with concurrent runs or controlled permits
|
||||
showing initial and repair generations never exceed the same backend limit
|
||||
and repairs do not perform a second admission.
|
||||
8. Extend the public error sentinel contract test with
|
||||
`ErrCapacityExceeded`.
|
||||
|
||||
Avoid a second HTTP-level concurrency suite: the capacity client tests and one
|
||||
assembled injected-client workflow already protect the shared wrapper used by
|
||||
the built-in client.
|
||||
|
||||
### Focused Validation
|
||||
|
||||
Run:
|
||||
|
||||
```sh
|
||||
gofmt -w engine.go errors.go backends.go \
|
||||
internal/usecase/runner.go internal/usecase/runner_test.go \
|
||||
engine_test.go public_contract_test.go
|
||||
go test . ./internal/backend ./internal/capacity ./internal/usecase
|
||||
go test -race . ./internal/capacity ./internal/usecase
|
||||
go vet . ./internal/backend ./internal/capacity ./internal/usecase
|
||||
git diff --check
|
||||
```
|
||||
|
||||
Add any newly created capacity files to `gofmt` when they changed in this
|
||||
stage.
|
||||
|
||||
### Completion Gate
|
||||
|
||||
This stage is complete when every engine has independent pools, limited runs
|
||||
are bounded and FIFO at generation, endpoint routing is correct, capacity and
|
||||
context errors are stable, repairs reuse admission, and both client kinds pass
|
||||
through the same wrapper.
|
||||
|
||||
## Stage 5 — Durable Documentation And Final Validation
|
||||
|
||||
**Status:** Pending.
|
||||
|
||||
### Goal
|
||||
|
||||
Move implemented contracts into their durable owners, record compatibility
|
||||
impact, and validate the complete repository.
|
||||
|
||||
### Work
|
||||
|
||||
1. Review every changed exported declaration. Ensure GoDoc is the canonical
|
||||
owner of exact field types, nil/zero semantics, defaulting, error identity,
|
||||
engine scope, concurrency safety, cancellation, and source compatibility.
|
||||
2. Update `doc.go` so its concurrency summary acknowledges backend scheduling
|
||||
while continuing to require injected collaborators to be concurrency-safe.
|
||||
3. Update `docs/consumers/pkg-promptkit.md` with task-oriented examples for:
|
||||
- a limited local backend;
|
||||
- omitted queue capacity selecting 1024;
|
||||
- explicit zero queue capacity; and
|
||||
- handling `ErrCapacityExceeded`.
|
||||
Keep exact field semantics in GoDoc rather than duplicating a full table.
|
||||
4. Add `docs/internal/capacity.md` as the durable owner of pool lifecycle,
|
||||
admission, FIFO active permits, cancellation, client wrapping, and test
|
||||
ownership.
|
||||
5. Add `internal/capacity` to `docs/internal/overview.md`.
|
||||
6. Update `docs/policy/architecture.md` to include the implemented component
|
||||
and root assembly dependency without turning policy into an API reference.
|
||||
7. Update `docs/internal/runner.md` to describe the shared two-phase
|
||||
preparation pipeline, early bounded admission, lease lifetime, generation
|
||||
permits, repairs, capacity failures, and cancellation.
|
||||
8. Review `docs/formats.md`; add only a concise link or clarification if needed
|
||||
to explain that endpoint overrides preserve backend capacity identity.
|
||||
Do not add concurrency fields to YAML.
|
||||
9. Do not change the OpenAI-compatible integration contract or
|
||||
`docs/internal/llm.md` unless implementation changes their current
|
||||
statements; scheduling is outside the provider wire contract and concrete
|
||||
model-client implementation.
|
||||
10. Record in the implementation handoff that built-in OpenRouter now limits
|
||||
active generations to 16 with queue capacity 1024 and that the release
|
||||
must be a pre-`v1` minor release. Do not edit the release procedure or
|
||||
create a tag.
|
||||
11. After every check passes, set `concurrency.md`, this implementation plan,
|
||||
and each stage status to `Complete`. Do not remove the roadmaps in the
|
||||
implementation change; lifecycle retirement follows review.
|
||||
|
||||
### Full Validation
|
||||
|
||||
Run the complete sequence from `docs/development.md`:
|
||||
|
||||
```sh
|
||||
go test ./...
|
||||
go test -race ./...
|
||||
go vet ./...
|
||||
go build ./...
|
||||
go run ./examples/go-library/prepare
|
||||
gofmt -l $(git ls-files '*.go')
|
||||
git diff --check
|
||||
```
|
||||
|
||||
The formatting command must produce no paths. Follow every added or changed
|
||||
Markdown link and confirm its target and heading exist.
|
||||
|
||||
Also inspect:
|
||||
|
||||
```sh
|
||||
git status --short
|
||||
git diff --stat
|
||||
git diff
|
||||
```
|
||||
|
||||
Confirm that:
|
||||
|
||||
- only intended backend, capacity, runner, facade, test, documentation, and
|
||||
roadmap files changed;
|
||||
- no `go.work`, `go.work.sum`, local module replacement, credential,
|
||||
generated binary, coverage output, or unrelated change was introduced;
|
||||
- the built-in OpenRouter policy is exactly 16/1024;
|
||||
- custom and endpoint-only backends remain unlimited by omission;
|
||||
- explicit queue zero is distinguishable from omission;
|
||||
- no capacity value enters execution targets, generated requests, stable JSON,
|
||||
prompt/profile YAML, or provider payloads;
|
||||
- every engine owns distinct pools with no package-global mutable state;
|
||||
- every initial and repair generation uses the active permit wrapper;
|
||||
- capacity and waiter state is released on success, error, panic unwinding,
|
||||
and cancellation;
|
||||
- concurrency tests use deterministic coordination rather than sleeps;
|
||||
- current-state documentation describes implemented behavior rather than
|
||||
referring readers to the roadmaps; and
|
||||
- the feature and implementation roadmaps contain no unresolved work marked
|
||||
complete.
|
||||
|
||||
### Completion Gate
|
||||
|
||||
The implementation is complete only when every target-end-state item in
|
||||
`concurrency.md` is implemented, race-enabled tests demonstrate the configured
|
||||
limits and cancellation safety, durable contracts no longer depend on roadmap
|
||||
prose, and the OpenRouter compatibility change is clearly reported for the
|
||||
next minor release.
|
||||
|
||||
## Open Questions
|
||||
|
||||
None. The feature roadmap and this plan fix the public representation,
|
||||
registry defaults, admission bound, FIFO generation behavior, early-routing
|
||||
refactor, cancellation races, error identities, engine and repair lifetimes,
|
||||
test ownership, compatibility treatment, and non-goals required for
|
||||
implementation.
|
||||
179
engine.go
179
engine.go
@@ -12,7 +12,9 @@ import (
|
||||
"time"
|
||||
|
||||
artifactadapter "gitea.maximumdirect.net/eric/promptkit/internal/artifact"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile/builtin"
|
||||
@@ -22,42 +24,96 @@ import (
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/validate"
|
||||
)
|
||||
|
||||
// ErrInvalidConfig indicates invalid public engine configuration.
|
||||
// ErrInvalidConfig identifies invalid engine construction, including missing
|
||||
// required configuration, invalid options or backend registrations, and a nil
|
||||
// Engine receiver.
|
||||
var ErrInvalidConfig = errors.New("invalid engine configuration")
|
||||
|
||||
var (
|
||||
ErrInvalidRequest = errors.New("invalid run request")
|
||||
ErrPromptNotFound = errors.New("prompt not found")
|
||||
ErrProfileNotFound = errors.New("profile not found")
|
||||
ErrProfileRequired = errors.New("profile selection is required")
|
||||
ErrPromptLoad = errors.New("failed to load prompt definition")
|
||||
ErrProfileLoad = errors.New("failed to load execution profile")
|
||||
// ErrInvalidRequest identifies a request whose required values, overrides,
|
||||
// credentials, or effective settings are invalid.
|
||||
ErrInvalidRequest = errors.New("invalid run request")
|
||||
// ErrPromptNotFound identifies a requested prompt ID or version that is not
|
||||
// present in the selected prompt source. It does not also match
|
||||
// ErrPromptLoad.
|
||||
ErrPromptNotFound = errors.New("prompt not found")
|
||||
// ErrProfileNotFound identifies a selected profile ID that is absent from
|
||||
// every configured profile source. It does not also match ErrProfileLoad.
|
||||
ErrProfileNotFound = errors.New("profile not found")
|
||||
// ErrProfileRequired identifies a request for which neither RunRequest.ProfileID
|
||||
// nor the selected prompt's default profile is present. Such an error also
|
||||
// matches ErrInvalidRequest.
|
||||
ErrProfileRequired = errors.New("profile selection is required")
|
||||
// ErrPromptLoad identifies a failure to read, decode, validate, select, or
|
||||
// hash a prompt definition, except for the not-found case represented by
|
||||
// ErrPromptNotFound.
|
||||
ErrPromptLoad = errors.New("failed to load prompt definition")
|
||||
// ErrProfileLoad identifies a failure to read, decode, validate, or select
|
||||
// an execution profile or resolve its backend, except for the profile
|
||||
// not-found case represented by ErrProfileNotFound.
|
||||
ErrProfileLoad = errors.New("failed to load execution profile")
|
||||
// ErrAPIKeyEnvMissing identifies an APIKeyEnv whose environment variable is
|
||||
// unset or empty when no direct RunRequest.APIKey takes precedence. Such an
|
||||
// error also matches ErrInvalidRequest.
|
||||
ErrAPIKeyEnvMissing = errors.New("api_key_env points to an unset environment variable")
|
||||
ErrArtifactLoad = errors.New("failed to load artifact")
|
||||
ErrPromptRender = errors.New("failed to render prompt")
|
||||
ErrLLMGenerate = errors.New("failed to generate output")
|
||||
ErrValidation = errors.New("failed to validate output")
|
||||
// ErrArtifactLoad identifies a failure to resolve an input artifact. Errors
|
||||
// returned by an injected ArtifactReader remain available through errors.Is.
|
||||
ErrArtifactLoad = errors.New("failed to load artifact")
|
||||
// ErrPromptRender identifies a failure to render prompt messages or the
|
||||
// session ID from the resolved inputs and variables.
|
||||
ErrPromptRender = errors.New("failed to render prompt")
|
||||
// ErrLLMGenerate identifies a model-client failure or a nil successful
|
||||
// response. Errors returned by an injected LLMClient remain available
|
||||
// through errors.Is.
|
||||
ErrLLMGenerate = errors.New("failed to generate output")
|
||||
// ErrValidation identifies an operational failure to load or compile a
|
||||
// schema or validate output. A completed validation whose Status is
|
||||
// ValidationFailed is returned in RunResult without this error.
|
||||
ErrValidation = errors.New("failed to validate output")
|
||||
)
|
||||
|
||||
// Engine prepares and runs Promptkit prompt requests.
|
||||
//
|
||||
// An Engine is safe for concurrent calls to [Engine.Prepare] and [Engine.Run].
|
||||
// Injected collaborators may consequently be invoked concurrently.
|
||||
type Engine struct {
|
||||
runner *usecase.Runner
|
||||
}
|
||||
|
||||
// Config configures a public Promptkit engine.
|
||||
// Config selects the directory-backed sources and built-in model-client
|
||||
// transport used by [NewEngine]. Config has no stable JSON representation.
|
||||
type Config struct {
|
||||
PromptDir string
|
||||
// PromptDir is the directory searched recursively for prompt definitions.
|
||||
// It is required unless a WithPromptFS or WithPromptFile option supplies the
|
||||
// prompt source.
|
||||
PromptDir string
|
||||
// ProfileDir is an optional directory whose profiles take precedence over
|
||||
// embedded built-in profiles. An empty value selects only built-ins unless
|
||||
// profile options are also supplied.
|
||||
ProfileDir string
|
||||
SchemaDir string
|
||||
// SchemaDir is the root for JSON Schema files. An empty value uses the
|
||||
// current directory. WithSchemaFS or WithSchemaFile replaces this source.
|
||||
SchemaDir string
|
||||
// Timeout is the transport-wide safety cap for the built-in LLM client
|
||||
// when HTTPClient is absent or has a non-positive timeout.
|
||||
// when HTTPClient is absent or has a non-positive timeout. A zero or negative
|
||||
// value selects the 10-minute default.
|
||||
Timeout time.Duration
|
||||
// HTTPClient is cloned for the built-in LLM client. Its positive Timeout
|
||||
// takes precedence over Config.Timeout as the transport-wide safety cap.
|
||||
// takes precedence over Timeout. A zero or negative client Timeout inherits
|
||||
// Timeout or the 10-minute default. The supplied client is not mutated. This
|
||||
// field is ignored when WithLLMClient is used.
|
||||
HTTPClient *http.Client
|
||||
}
|
||||
|
||||
// Option customizes engine construction.
|
||||
//
|
||||
// NewEngine applies options in argument order and ignores nil options. Within
|
||||
// each prompt-source, profile-source, in-memory-profile, schema-source,
|
||||
// model-client, and artifact-reader category, the last non-nil valid option
|
||||
// replaces earlier options in that category. WithBackend is the additive
|
||||
// exception: unique registrations accumulate, and a repeated backend ID is an
|
||||
// error rather than a replacement. An invalid option fails construction even
|
||||
// if a later option would replace it.
|
||||
type Option interface {
|
||||
apply(*engineOptions) error
|
||||
}
|
||||
@@ -74,6 +130,7 @@ type engineOptions struct {
|
||||
promptDefs promptdef.Repository
|
||||
profiles profile.Repository
|
||||
memoryProfiles profile.Repository
|
||||
backends []domain.Backend
|
||||
validator validate.Validator
|
||||
promptSource bool
|
||||
profileSource bool
|
||||
@@ -82,7 +139,10 @@ type engineOptions struct {
|
||||
artifactSource bool
|
||||
}
|
||||
|
||||
// WithLLMClient injects a custom LLM client for execution.
|
||||
// WithLLMClient replaces the built-in model client used by [Engine.Run].
|
||||
//
|
||||
// A nil client makes NewEngine fail with ErrInvalidConfig. The client may be
|
||||
// called concurrently and is not used by [Engine.Prepare].
|
||||
func WithLLMClient(client LLMClient) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
if client == nil {
|
||||
@@ -93,7 +153,11 @@ func WithLLMClient(client LLMClient) Option {
|
||||
})
|
||||
}
|
||||
|
||||
// WithArtifactReader injects a reader for every input artifact reference.
|
||||
// WithArtifactReader replaces the default reader for every input artifact
|
||||
// reference, regardless of its ArtifactRef.Type.
|
||||
//
|
||||
// A nil reader makes NewEngine fail with ErrInvalidConfig. The reader may be
|
||||
// called concurrently.
|
||||
func WithArtifactReader(reader ArtifactReader) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
if reader == nil {
|
||||
@@ -109,6 +173,9 @@ func WithArtifactReader(reader ArtifactReader) Option {
|
||||
//
|
||||
// The source uses the same strict prompt YAML rules as configured prompt
|
||||
// directories, and prompt content_file paths resolve within this source.
|
||||
// fsys must be non-nil and root must be non-empty; otherwise NewEngine fails
|
||||
// with ErrInvalidConfig. This option replaces Config.PromptDir and earlier
|
||||
// prompt-source options.
|
||||
func WithPromptFS(fsys fs.FS, root string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
if fsys == nil {
|
||||
@@ -125,7 +192,9 @@ func WithPromptFS(fsys fs.FS, root string) Option {
|
||||
|
||||
// WithPromptFile loads prompt definitions from the single prompt file at path.
|
||||
//
|
||||
// Relative prompt content_file paths resolve from the file's directory.
|
||||
// Relative prompt content_file paths resolve from the file's directory. path
|
||||
// must name an existing non-directory file when NewEngine applies the option.
|
||||
// This option replaces Config.PromptDir and earlier prompt-source options.
|
||||
func WithPromptFile(path string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
fsys, root, err := fileSource(path)
|
||||
@@ -142,6 +211,10 @@ func WithPromptFile(path string) Option {
|
||||
//
|
||||
// Profiles from this source overlay built-in profiles. Profile YAML must use
|
||||
// api_key_env for environment-based credentials; raw API keys are rejected.
|
||||
// fsys must be non-nil and root must be non-empty; otherwise NewEngine fails
|
||||
// with ErrInvalidConfig. This option replaces Config.ProfileDir and earlier
|
||||
// file or FS profile-source options, but remains below WithProfiles in
|
||||
// precedence.
|
||||
func WithProfileFS(fsys fs.FS, root string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
if fsys == nil {
|
||||
@@ -159,7 +232,10 @@ func WithProfileFS(fsys fs.FS, root string) Option {
|
||||
// WithProfileFile loads execution profiles from the single profile file at path.
|
||||
//
|
||||
// The profile overlays built-in profiles. Profile YAML must use api_key_env for
|
||||
// environment-based credentials; raw API keys are rejected.
|
||||
// environment-based credentials; raw API keys are rejected. path must name an
|
||||
// existing non-directory file when NewEngine applies the option. This option
|
||||
// replaces Config.ProfileDir and earlier file or FS profile-source options,
|
||||
// but remains below WithProfiles in precedence.
|
||||
func WithProfileFile(path string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
fsys, root, err := fileSource(path)
|
||||
@@ -174,6 +250,11 @@ func WithProfileFile(path string) Option {
|
||||
|
||||
// WithProfiles configures in-memory profiles that take precedence over
|
||||
// configured profile files and built-in profiles.
|
||||
//
|
||||
// NewEngine validates and copies every profile. IDs must be unique within one
|
||||
// call. An invalid profile, duplicate ID, or unsupported ExtraParams value
|
||||
// makes construction fail with ErrInvalidConfig. Repeating WithProfiles
|
||||
// replaces the complete earlier in-memory set rather than merging it.
|
||||
func WithProfiles(profiles ...Profile) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
repo, err := newMemoryProfileRepository(profiles)
|
||||
@@ -189,7 +270,9 @@ func WithProfiles(profiles ...Profile) Option {
|
||||
// WithSchemaFS loads JSON Schema documents from fsys under root.
|
||||
//
|
||||
// Prompt schema_path values resolve within this source when schema validation
|
||||
// or structured output is requested.
|
||||
// or structured output is requested. fsys must be non-nil and root must be
|
||||
// non-empty; otherwise NewEngine fails with ErrInvalidConfig. This option
|
||||
// replaces Config.SchemaDir and earlier schema-source options.
|
||||
func WithSchemaFS(fsys fs.FS, root string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
if fsys == nil {
|
||||
@@ -206,7 +289,9 @@ func WithSchemaFS(fsys fs.FS, root string) Option {
|
||||
|
||||
// WithSchemaFile loads JSON Schema documents from the single schema file at path.
|
||||
//
|
||||
// Prompt schema_path values refer to the file's base name.
|
||||
// Prompt schema_path values refer to the file's base name. path must name an
|
||||
// existing non-directory file when NewEngine applies the option. This option
|
||||
// replaces Config.SchemaDir and earlier schema-source options.
|
||||
func WithSchemaFile(path string) Option {
|
||||
return optionFunc(func(options *engineOptions) error {
|
||||
fsys, root, err := fileSource(path)
|
||||
@@ -220,6 +305,16 @@ func WithSchemaFile(path string) Option {
|
||||
}
|
||||
|
||||
// NewEngine constructs an Engine from configuration and options.
|
||||
//
|
||||
// Options are applied in order according to [Option]. PromptDir is required
|
||||
// unless a prompt-source option is present. Construction validates option
|
||||
// arguments, in-memory profiles, and backend registrations but defers reading
|
||||
// and validating prompt, file-backed profile, and schema contents until Prepare
|
||||
// or Run needs them.
|
||||
//
|
||||
// NewEngine returns an error matching ErrInvalidConfig for invalid
|
||||
// configuration or options. It does not perform model requests or require
|
||||
// credentials.
|
||||
func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
||||
var options engineOptions
|
||||
for _, opt := range opts {
|
||||
@@ -247,6 +342,11 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
||||
profiles = profile.NewOverlayRepository(options.memoryProfiles, profiles)
|
||||
}
|
||||
|
||||
backendRegistry, err := backend.NewRegistry(options.backends)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: failed to construct backend registry: %v", ErrInvalidConfig, err)
|
||||
}
|
||||
|
||||
validator := options.validator
|
||||
if !options.validatorSource {
|
||||
schemaDir := cfg.SchemaDir
|
||||
@@ -277,6 +377,7 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
||||
runner: usecase.NewRunner(
|
||||
promptDefs,
|
||||
profiles,
|
||||
backendRegistry,
|
||||
artifacts,
|
||||
prompt.NewGoRenderer(),
|
||||
llmClient,
|
||||
@@ -305,7 +406,23 @@ func fileSource(name string) (fs.FS, string, error) {
|
||||
return os.DirFS(dir), filepath.ToSlash(base), nil
|
||||
}
|
||||
|
||||
// Prepare resolves a prompt request without calling an LLM.
|
||||
// Prepare resolves and renders a prompt request without calling an LLM.
|
||||
//
|
||||
// Prepare selects the prompt and profile, resolves any selected backend and
|
||||
// effective execution settings, resolves the output contract, loads and hashes
|
||||
// inputs, loads structured-output schema metadata when required, and renders
|
||||
// the session ID and messages. The returned PreparedRun is owned by the caller
|
||||
// and never contains a resolved API-key value, model output, or validation
|
||||
// result.
|
||||
//
|
||||
// A nil Engine returns an error matching ErrInvalidConfig. Request and
|
||||
// preparation failures may match ErrInvalidRequest, ErrPromptNotFound,
|
||||
// ErrPromptLoad, ErrProfileNotFound, ErrProfileLoad, ErrProfileRequired,
|
||||
// ErrAPIKeyEnvMissing, ErrArtifactLoad, ErrPromptRender, or ErrValidation as
|
||||
// applicable. Cancellation is passed to the active collaborator and is
|
||||
// reported in the applicable operation category; no general errors.Is
|
||||
// relationship to ctx.Err is promised. Prepare returns no partial result on
|
||||
// error.
|
||||
func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, error) {
|
||||
if e == nil || e.runner == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
@@ -323,7 +440,21 @@ func (e *Engine) Prepare(ctx context.Context, req RunRequest) (*PreparedRun, err
|
||||
return fromDomainPreparedRun(prepared), nil
|
||||
}
|
||||
|
||||
// Run executes a prompt request and returns the generated artifact and metadata.
|
||||
// Run prepares a request, invokes the configured LLMClient, and validates the
|
||||
// generated output.
|
||||
//
|
||||
// A content-validation failure is a successful run whose
|
||||
// RunResult.Validation has Status ValidationFailed. An inability to perform
|
||||
// validation returns an error matching ErrValidation and no partial result.
|
||||
// The public Engine does not perform output repair, so validation is
|
||||
// single-pass even when OutputContract.RepairAttempts is positive.
|
||||
//
|
||||
// Run can return every error category documented by [Engine.Prepare], plus
|
||||
// ErrLLMGenerate. Errors from injected clients remain available through
|
||||
// errors.Is. Cancellation is passed through the active collaborator and is
|
||||
// reported in the applicable operation category; no general errors.Is
|
||||
// relationship to ctx.Err is promised. A nil Engine returns ErrInvalidConfig.
|
||||
// Run returns no partial result on error.
|
||||
func (e *Engine) Run(ctx context.Context, req RunRequest) (*RunResult, error) {
|
||||
if e == nil || e.runner == nil {
|
||||
return nil, fmt.Errorf("%w: engine is nil", ErrInvalidConfig)
|
||||
|
||||
243
engine_test.go
243
engine_test.go
@@ -281,6 +281,9 @@ func TestEngineExecutionSettingPrecedence(t *testing.T) {
|
||||
intPointer := func(value int) *int {
|
||||
return &value
|
||||
}
|
||||
stringPointer := func(value string) *string {
|
||||
return &value
|
||||
}
|
||||
|
||||
defaultsProfile := executionProfileFixture{
|
||||
id: "settings-defaults",
|
||||
@@ -388,7 +391,7 @@ func TestEngineExecutionSettingPrecedence(t *testing.T) {
|
||||
TopP: floatPointer(requestTarget.TopP),
|
||||
TimeoutSeconds: intPointer(requestTarget.TimeoutSeconds),
|
||||
ServiceTier: requestTarget.ServiceTier,
|
||||
ReasoningEffort: requestTarget.ReasoningEffort,
|
||||
ReasoningEffort: stringPointer(requestTarget.ReasoningEffort),
|
||||
APIKeyEnv: requestTarget.APIKeyEnv,
|
||||
ExtraParams: requestTarget.ExtraParams,
|
||||
},
|
||||
@@ -452,6 +455,43 @@ func TestEngineExecutionSettingPrecedence(t *testing.T) {
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("blank request reasoning clears profile setting", func(t *testing.T) {
|
||||
profile := executionProfileFixture{
|
||||
id: "settings-reasoning-clear",
|
||||
endpoint: "http://profile-reasoning.test/v1",
|
||||
model: "profile-reasoning-model",
|
||||
reasoningEffort: "medium",
|
||||
}
|
||||
profileDir := t.TempDir()
|
||||
writeExecutionProfileFixture(t, profileDir, profile)
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
PromptDir: frameworkPromptDir,
|
||||
ProfileDir: profileDir,
|
||||
SchemaDir: frameworkSchemaDir,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: frameworkMarkdownSummaryPromptID,
|
||||
ProfileID: profile.id,
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline("Nia labels the archive."),
|
||||
"glossary": promptkit.Inline("archive: A catalogued collection."),
|
||||
},
|
||||
Execution: &promptkit.ExecutionTargetOverride{
|
||||
ReasoningEffort: stringPointer(" \t "),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare engine: %v", err)
|
||||
}
|
||||
if prepared.EffectiveModelParams.ReasoningEffort != "" {
|
||||
t.Fatalf("expected blank request reasoning to clear profile value, got %q", prepared.EffectiveModelParams.ReasoningEffort)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunSucceedsWithInjectedLLMClient(t *testing.T) {
|
||||
@@ -571,19 +611,26 @@ func TestEngineRunWithDirectorySourcesAndFileInputs(t *testing.T) {
|
||||
|
||||
func TestRunPassesPreparedRequestToInjectedLLMClient(t *testing.T) {
|
||||
const directKey = "direct-injected-key"
|
||||
const directSession = "assembled-session"
|
||||
fake := &fakeLLMClient{
|
||||
response: &promptkit.GenerateResponse{Content: `{"events":[{"title":"Archive labelled"}]}`},
|
||||
}
|
||||
engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(fake))
|
||||
|
||||
_, err := engine.Run(context.Background(), promptkit.RunRequest{
|
||||
PromptID: frameworkStructuredEventsPromptID,
|
||||
APIKey: directKey,
|
||||
runRequest := promptkit.RunRequest{
|
||||
PromptID: frameworkStructuredEventsPromptID,
|
||||
SessionID: " " + directSession + " ",
|
||||
APIKey: directKey,
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline("Rin opens the gate."),
|
||||
"glossary": promptkit.Inline("gate: A guarded passage."),
|
||||
},
|
||||
})
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), runRequest)
|
||||
if err != nil {
|
||||
t.Fatalf("expected prepare to succeed, got %v", err)
|
||||
}
|
||||
result, err := engine.Run(context.Background(), runRequest)
|
||||
if err != nil {
|
||||
t.Fatalf("expected run to succeed, got %v", err)
|
||||
}
|
||||
@@ -594,6 +641,16 @@ func TestRunPassesPreparedRequestToInjectedLLMClient(t *testing.T) {
|
||||
if len(req.Prompt.Messages) != 2 || !strings.Contains(req.Prompt.Messages[1].Content, "Rin opens the gate.") {
|
||||
t.Fatalf("expected rendered prompt in generate request, got %+v", req.Prompt)
|
||||
}
|
||||
if prepared.SessionID != directSession ||
|
||||
req.Prompt.SessionID != directSession ||
|
||||
result.SessionID != directSession {
|
||||
t.Fatalf(
|
||||
"direct session did not propagate consistently: prepared=%q generated=%q result=%q",
|
||||
prepared.SessionID,
|
||||
req.Prompt.SessionID,
|
||||
result.SessionID,
|
||||
)
|
||||
}
|
||||
if req.StructuredOutput == nil || req.StructuredOutput.Type != promptkit.StructuredOutputJSONSchema || req.StructuredOutput.JSONSchema == nil {
|
||||
t.Fatalf("expected structured output handoff, got %+v", req.StructuredOutput)
|
||||
}
|
||||
@@ -755,6 +812,92 @@ func TestRunUsesDirectAPIKeyWithDefaultLLMClient(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunUsesResolvedBackendWithBuiltInLLMClient(t *testing.T) {
|
||||
const (
|
||||
backendID = "local-test"
|
||||
envName = "PROMPTKIT_BACKEND_TRANSPORT_KEY"
|
||||
apiKey = "synthetic-backend-key"
|
||||
)
|
||||
t.Setenv(envName, apiKey)
|
||||
|
||||
var (
|
||||
gotAuth string
|
||||
gotBody map[string]any
|
||||
)
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotAuth = r.Header.Get("Authorization")
|
||||
if r.URL.Path != "/v1/chat/completions" {
|
||||
t.Errorf("unexpected path: %s", r.URL.Path)
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&gotBody); err != nil {
|
||||
t.Errorf("decode request body: %v", err)
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
_, _ = w.Write([]byte(`{
|
||||
"choices": [{"message": {"role": "assistant", "content": "# Summary\n\nDone."}}],
|
||||
"usage": {"prompt_tokens": 3, "completion_tokens": 4, "total_tokens": 7}
|
||||
}`))
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{
|
||||
PromptDir: frameworkPromptDir,
|
||||
SchemaDir: frameworkSchemaDir,
|
||||
},
|
||||
promptkit.WithBackend(promptkit.Backend{
|
||||
ID: backendID,
|
||||
Endpoint: server.URL + "/v1",
|
||||
APIKeyEnv: envName,
|
||||
ExtraParams: map[string]any{
|
||||
"provider": "synthetic",
|
||||
},
|
||||
}),
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "backend-transport",
|
||||
BackendID: backendID,
|
||||
Model: "test-model",
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
|
||||
result, err := engine.Run(context.Background(), promptkit.RunRequest{
|
||||
PromptID: frameworkMarkdownSummaryPromptID,
|
||||
ProfileID: "backend-transport",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline("Rin opens the gate."),
|
||||
"glossary": promptkit.Inline("gate: A guarded passage."),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("run with resolved backend: %v", err)
|
||||
}
|
||||
if gotAuth != "Bearer "+apiKey {
|
||||
t.Fatalf("unexpected Authorization header: %q", gotAuth)
|
||||
}
|
||||
if gotBody["model"] != "test-model" || gotBody["provider"] != "synthetic" {
|
||||
t.Fatalf("backend defaults did not reach provider payload: %#v", gotBody)
|
||||
}
|
||||
for _, field := range []string{"backend_id", "api_key_env"} {
|
||||
if _, ok := gotBody[field]; ok {
|
||||
t.Fatalf("internal metadata field %q was serialized to provider payload: %#v", field, gotBody)
|
||||
}
|
||||
}
|
||||
bodyJSON, err := json.Marshal(gotBody)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal captured provider payload: %v", err)
|
||||
}
|
||||
if strings.Contains(string(bodyJSON), apiKey) {
|
||||
t.Fatalf("credential value was serialized to provider payload: %s", bodyJSON)
|
||||
}
|
||||
if result.SelectedBackendID != backendID ||
|
||||
result.EffectiveModelParams.Endpoint != server.URL+"/v1" ||
|
||||
result.EffectiveModelParams.APIKeyEnv != envName {
|
||||
t.Fatalf("unexpected resolved backend metadata: %+v", result)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPrepareDirectAPIKeyBypassesMissingEnvWithoutLeakingOrHashing(t *testing.T) {
|
||||
const missingEnv = "PROMPTKIT_PUBLIC_PREPARE_MISSING"
|
||||
const firstKey = "first-direct-key"
|
||||
@@ -1020,6 +1163,7 @@ func TestPublicErrorsSupportErrorsIs(t *testing.T) {
|
||||
client promptkit.LLMClient
|
||||
schemaDir string
|
||||
want error
|
||||
notWant error
|
||||
}{
|
||||
{
|
||||
name: "invalid request",
|
||||
@@ -1028,10 +1172,11 @@ func TestPublicErrorsSupportErrorsIs(t *testing.T) {
|
||||
want: promptkit.ErrInvalidRequest,
|
||||
},
|
||||
{
|
||||
name: "prompt not found",
|
||||
req: promptkit.RunRequest{PromptID: "missing.prompt"},
|
||||
client: &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}},
|
||||
want: promptkit.ErrPromptNotFound,
|
||||
name: "prompt not found",
|
||||
req: promptkit.RunRequest{PromptID: "missing.prompt"},
|
||||
client: &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}},
|
||||
want: promptkit.ErrPromptNotFound,
|
||||
notWant: promptkit.ErrPromptLoad,
|
||||
},
|
||||
{
|
||||
name: "profile not found",
|
||||
@@ -1042,8 +1187,9 @@ func TestPublicErrorsSupportErrorsIs(t *testing.T) {
|
||||
"transcript": promptkit.Inline("Rin opens the gate."),
|
||||
},
|
||||
},
|
||||
client: &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}},
|
||||
want: promptkit.ErrProfileNotFound,
|
||||
client: &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}},
|
||||
want: promptkit.ErrProfileNotFound,
|
||||
notWant: promptkit.ErrProfileLoad,
|
||||
},
|
||||
{
|
||||
name: "artifact load",
|
||||
@@ -1114,6 +1260,9 @@ func TestPublicErrorsSupportErrorsIs(t *testing.T) {
|
||||
if !errors.Is(err, tc.want) {
|
||||
t.Fatalf("expected errors.Is(%v), got %v", tc.want, err)
|
||||
}
|
||||
if tc.notWant != nil && errors.Is(err, tc.notWant) {
|
||||
t.Fatalf("did not expect errors.Is(%v), got %v", tc.notWant, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1265,6 +1414,18 @@ func TestPrepareUsesBuiltInProfileWithoutProfileDir(t *testing.T) {
|
||||
if prepared.SelectedProfileID != "mistral-small-3" {
|
||||
t.Fatalf("unexpected selected profile: %q", prepared.SelectedProfileID)
|
||||
}
|
||||
if prepared.SelectedBackendID != promptkit.BackendOpenRouter {
|
||||
t.Fatalf("unexpected selected backend: %q", prepared.SelectedBackendID)
|
||||
}
|
||||
if prepared.EffectiveModelParams.BackendID != promptkit.BackendOpenRouter {
|
||||
t.Fatalf("unexpected effective backend: %q", prepared.EffectiveModelParams.BackendID)
|
||||
}
|
||||
if prepared.EffectiveModelParams.Endpoint != "https://openrouter.ai/api/v1" {
|
||||
t.Fatalf("unexpected built-in endpoint: %q", prepared.EffectiveModelParams.Endpoint)
|
||||
}
|
||||
if prepared.EffectiveModelParams.APIKeyEnv != "OPENROUTER_API_KEY" {
|
||||
t.Fatalf("unexpected built-in api key environment name: %q", prepared.EffectiveModelParams.APIKeyEnv)
|
||||
}
|
||||
if prepared.EffectiveModelParams.Model != "mistralai/mistral-small-3.2-24b-instruct" {
|
||||
t.Fatalf("unexpected built-in model: %q", prepared.EffectiveModelParams.Model)
|
||||
}
|
||||
@@ -1635,6 +1796,7 @@ func TestOpenAICompatibleProfileRunsThroughNormalProfilePath(t *testing.T) {
|
||||
fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
|
||||
prof := promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||
ID: "template-profile",
|
||||
BackendID: " openrouter ",
|
||||
Endpoint: "http://template/v1",
|
||||
Model: "template-model",
|
||||
APIKeyRequired: true,
|
||||
@@ -1666,7 +1828,9 @@ func TestOpenAICompatibleProfileRunsThroughNormalProfilePath(t *testing.T) {
|
||||
if len(fake.requests) != 1 {
|
||||
t.Fatalf("expected one request, got %d", len(fake.requests))
|
||||
}
|
||||
if fake.requests[0].Target.Model != "template-model" || fake.requests[0].APIKey != "template-key" {
|
||||
if fake.requests[0].Target.BackendID != promptkit.BackendOpenRouter ||
|
||||
fake.requests[0].Target.Model != "template-model" ||
|
||||
fake.requests[0].APIKey != "template-key" {
|
||||
t.Fatalf("unexpected generated request: %+v", fake.requests[0])
|
||||
}
|
||||
if !reflect.DeepEqual(fake.requests[0].Target.ExtraParams, map[string]any{"provider": "template"}) {
|
||||
@@ -1699,6 +1863,16 @@ func TestEngineRunLayersTransportAndGenerationTimeouts(t *testing.T) {
|
||||
configTimeout: 5 * time.Second,
|
||||
wantRemainingAtRequest: 5 * time.Second,
|
||||
},
|
||||
{
|
||||
name: "zero configuration uses ten minute transport default",
|
||||
wantRemainingAtRequest: 10 * time.Minute,
|
||||
},
|
||||
{
|
||||
name: "negative configuration uses ten minute transport default",
|
||||
configTimeout: -2 * time.Second,
|
||||
suppliedClientTimeout: -3 * time.Second,
|
||||
wantRemainingAtRequest: 10 * time.Minute,
|
||||
},
|
||||
{
|
||||
name: "profile deadline is shorter than transport cap",
|
||||
suppliedClientTimeout: 6 * time.Second,
|
||||
@@ -2240,6 +2414,11 @@ func TestExtraParamsTypedNestedValuesAreCopiedAcrossPublicBoundary(t *testing.T)
|
||||
}
|
||||
|
||||
func TestRunRejectsInvalidExtraParams(t *testing.T) {
|
||||
cyclicMap := map[string]any{}
|
||||
cyclicMap["self"] = cyclicMap
|
||||
cyclicSlice := []any{nil}
|
||||
cyclicSlice[0] = cyclicSlice
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
extraParams map[string]any
|
||||
@@ -2251,42 +2430,10 @@ func TestRunRejectsInvalidExtraParams(t *testing.T) {
|
||||
{name: "nan", extraParams: map[string]any{"bad": math.NaN()}},
|
||||
{name: "positive infinity", extraParams: map[string]any{"bad": math.Inf(1)}},
|
||||
{name: "negative infinity", extraParams: map[string]any{"bad": math.Inf(-1)}},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
|
||||
engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(fake))
|
||||
|
||||
_, err := engine.Run(context.Background(), promptkit.RunRequest{
|
||||
PromptID: frameworkMarkdownSummaryPromptID,
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline("Rin opens the gate."),
|
||||
"glossary": promptkit.Inline("gate: A guarded passage."),
|
||||
},
|
||||
Execution: &promptkit.ExecutionTargetOverride{ExtraParams: tc.extraParams},
|
||||
})
|
||||
if !errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
||||
}
|
||||
if len(fake.requests) != 0 {
|
||||
t.Fatalf("expected invalid request to fail before LLM call, got %d requests", len(fake.requests))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunRejectsCyclicExtraParams(t *testing.T) {
|
||||
cyclicMap := map[string]any{}
|
||||
cyclicMap["self"] = cyclicMap
|
||||
cyclicSlice := []any{nil}
|
||||
cyclicSlice[0] = cyclicSlice
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
extraParams map[string]any
|
||||
}{
|
||||
{name: "map", extraParams: cyclicMap},
|
||||
{name: "slice", extraParams: map[string]any{"cycle": cyclicSlice}},
|
||||
{name: "cyclic map", extraParams: cyclicMap},
|
||||
{name: "cyclic slice", extraParams: map[string]any{"cycle": cyclicSlice}},
|
||||
{name: "malformed JSON number", extraParams: map[string]any{"value": json.Number("+1")}},
|
||||
{name: "empty nested key", extraParams: map[string]any{"nested": map[string]any{"": true}}},
|
||||
}
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
|
||||
81
examples/go-library/run/main.go
Normal file
81
examples/go-library/run/main.go
Normal file
@@ -0,0 +1,81 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
type deterministicClient struct{}
|
||||
|
||||
func (deterministicClient) Generate(
|
||||
ctx context.Context,
|
||||
_ promptkit.GenerateRequest,
|
||||
) (*promptkit.GenerateResponse, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &promptkit.GenerateResponse{
|
||||
Content: "Ada finished the migration review.",
|
||||
Usage: promptkit.TokenUsage{
|
||||
PromptTokens: 12,
|
||||
CompletionTokens: 6,
|
||||
TotalTokens: 18,
|
||||
},
|
||||
}, nil
|
||||
}
|
||||
|
||||
type summary struct {
|
||||
Output string `json:"output"`
|
||||
ValidationStatus promptkit.ValidationStatus `json:"validation_status"`
|
||||
IsValid bool `json:"is_valid"`
|
||||
Model string `json:"model"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
}
|
||||
|
||||
func main() {
|
||||
engine, err := promptkit.NewEngine(
|
||||
promptkit.Config{},
|
||||
promptkit.WithPromptFile("examples/go-library/run/prompt.yaml"),
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "offline-example",
|
||||
Endpoint: "https://example.invalid/v1",
|
||||
Model: "offline-model",
|
||||
}),
|
||||
promptkit.WithLLMClient(deterministicClient{}),
|
||||
)
|
||||
if err != nil {
|
||||
exit(err)
|
||||
}
|
||||
|
||||
result, err := engine.Run(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "example.run",
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"note": promptkit.Inline("Ada finished the migration review."),
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
exit(err)
|
||||
}
|
||||
|
||||
encoder := json.NewEncoder(os.Stdout)
|
||||
encoder.SetIndent("", " ")
|
||||
if err := encoder.Encode(summary{
|
||||
Output: result.RawOutput,
|
||||
ValidationStatus: result.Validation.Status,
|
||||
IsValid: result.Validation.IsValid,
|
||||
Model: result.ModelName,
|
||||
TotalTokens: result.Usage.TotalTokens,
|
||||
}); err != nil {
|
||||
exit(err)
|
||||
}
|
||||
}
|
||||
|
||||
func exit(err error) {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
os.Exit(1)
|
||||
}
|
||||
16
examples/go-library/run/prompt.yaml
Normal file
16
examples/go-library/run/prompt.yaml
Normal file
@@ -0,0 +1,16 @@
|
||||
id: example.run
|
||||
version: "1.0.0"
|
||||
default_profile: offline-example
|
||||
description: Run a prompt with a deterministic injected model client.
|
||||
inputs:
|
||||
- name: note
|
||||
required: true
|
||||
content_type: text/plain
|
||||
messages:
|
||||
- role: system
|
||||
content: Summarize the note in one sentence.
|
||||
- role: user
|
||||
content: '{{input "note"}}'
|
||||
output:
|
||||
format: text
|
||||
validation_mode: basic
|
||||
@@ -2,19 +2,23 @@ package promptkit
|
||||
|
||||
import "fmt"
|
||||
|
||||
// String returns a concise request summary without exposing direct API keys.
|
||||
// String returns a concise request summary without exposing the direct API key
|
||||
// or input and variable contents. Reflection-based formatting does not carry
|
||||
// this guarantee.
|
||||
func (r RunRequest) String() string {
|
||||
return r.redactedString()
|
||||
}
|
||||
|
||||
// GoString returns a concise request summary without exposing direct API keys.
|
||||
// GoString returns a concise request summary without exposing the direct API
|
||||
// key or input and variable contents. Reflection-based formatting does not
|
||||
// carry this guarantee.
|
||||
func (r RunRequest) GoString() string {
|
||||
return r.redactedString()
|
||||
}
|
||||
|
||||
func (r RunRequest) redactedString() string {
|
||||
return fmt.Sprintf(
|
||||
"promptkit.RunRequest{PromptID:%q PromptVersion:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t Metadata:%d}",
|
||||
"promptkit.RunRequest{PromptID:%q PromptVersion:%q ProfileID:%q APIKeySet:%t Inputs:%d Vars:%d ExecutionSet:%t ValidationSet:%t}",
|
||||
r.PromptID,
|
||||
r.PromptVersion,
|
||||
r.ProfileID,
|
||||
@@ -23,18 +27,19 @@ func (r RunRequest) redactedString() string {
|
||||
len(r.Vars),
|
||||
r.Execution != nil,
|
||||
r.Validation != nil,
|
||||
len(r.Metadata),
|
||||
)
|
||||
}
|
||||
|
||||
// String returns a concise request summary without exposing direct API keys or
|
||||
// rendered prompt content.
|
||||
// rendered prompt content. Reflection-based formatting does not carry this
|
||||
// guarantee.
|
||||
func (r GenerateRequest) String() string {
|
||||
return r.redactedString()
|
||||
}
|
||||
|
||||
// GoString returns a concise request summary without exposing direct API keys or
|
||||
// rendered prompt content.
|
||||
// rendered prompt content. Reflection-based formatting does not carry this
|
||||
// guarantee.
|
||||
func (r GenerateRequest) GoString() string {
|
||||
return r.redactedString()
|
||||
}
|
||||
|
||||
155
internal/backend/registry.go
Normal file
155
internal/backend/registry.go
Normal file
@@ -0,0 +1,155 @@
|
||||
// Package backend owns validated, immutable OpenAI-compatible backend
|
||||
// definitions.
|
||||
package backend
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||
)
|
||||
|
||||
const (
|
||||
// OpenRouterID is the reserved ID of Promptkit's built-in OpenRouter
|
||||
// backend.
|
||||
OpenRouterID = "openrouter"
|
||||
|
||||
openRouterEndpoint = "https://openrouter.ai/api/v1"
|
||||
openRouterAPIKeyEnv = "OPENROUTER_API_KEY"
|
||||
)
|
||||
|
||||
// ErrBackendNotFound identifies a registry lookup for an unknown backend ID.
|
||||
var ErrBackendNotFound = errors.New("backend not found")
|
||||
|
||||
var environmentVariableName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
|
||||
|
||||
// Registry is an immutable collection of validated backend definitions.
|
||||
type Registry struct {
|
||||
backends map[string]domain.Backend
|
||||
}
|
||||
|
||||
// NewRegistry constructs a registry containing the built-in OpenRouter
|
||||
// definition followed by the supplied additions. Every ID must be unique.
|
||||
func NewRegistry(additions []domain.Backend) (*Registry, error) {
|
||||
registry := &Registry{
|
||||
backends: make(map[string]domain.Backend, len(additions)+1),
|
||||
}
|
||||
|
||||
definitions := make([]domain.Backend, 0, len(additions)+1)
|
||||
definitions = append(definitions, domain.Backend{
|
||||
ID: OpenRouterID,
|
||||
Endpoint: openRouterEndpoint,
|
||||
APIKeyEnv: openRouterAPIKeyEnv,
|
||||
})
|
||||
definitions = append(definitions, additions...)
|
||||
|
||||
for _, definition := range definitions {
|
||||
definition.ID = strings.TrimSpace(definition.ID)
|
||||
if definition.ID == "" {
|
||||
return nil, errors.New("backend ID must not be blank")
|
||||
}
|
||||
if _, exists := registry.backends[definition.ID]; exists {
|
||||
return nil, fmt.Errorf("backend ID %q is already registered", definition.ID)
|
||||
}
|
||||
|
||||
normalized, err := normalizeBackend(definition)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
registry.backends[normalized.ID] = normalized
|
||||
}
|
||||
|
||||
return registry, nil
|
||||
}
|
||||
|
||||
// GetBackend returns a defensive copy of the backend registered with id.
|
||||
func (r *Registry) GetBackend(id string) (domain.Backend, error) {
|
||||
if r == nil {
|
||||
return domain.Backend{}, fmt.Errorf("%w: %q", ErrBackendNotFound, id)
|
||||
}
|
||||
definition, ok := r.backends[id]
|
||||
if !ok {
|
||||
return domain.Backend{}, fmt.Errorf("%w: %q", ErrBackendNotFound, id)
|
||||
}
|
||||
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
|
||||
if err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("copy backend %q: %w", id, err)
|
||||
}
|
||||
definition.ExtraParams = extraParams
|
||||
return definition, nil
|
||||
}
|
||||
|
||||
func normalizeBackend(definition domain.Backend) (domain.Backend, error) {
|
||||
definition.Endpoint = strings.TrimSpace(definition.Endpoint)
|
||||
if err := validateEndpoint(definition.Endpoint); err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("backend %q endpoint: %w", definition.ID, err)
|
||||
}
|
||||
|
||||
definition.APIKeyEnv = strings.TrimSpace(definition.APIKeyEnv)
|
||||
if definition.APIKeyEnv != "" && !environmentVariableName.MatchString(definition.APIKeyEnv) {
|
||||
return domain.Backend{}, fmt.Errorf(
|
||||
"backend %q api key environment variable %q is invalid",
|
||||
definition.ID,
|
||||
definition.APIKeyEnv,
|
||||
)
|
||||
}
|
||||
|
||||
keys := make([]string, 0, len(definition.ExtraParams))
|
||||
for key := range definition.ExtraParams {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, key := range keys {
|
||||
if key == "" {
|
||||
return domain.Backend{}, fmt.Errorf("backend %q extra parameter key must not be empty", definition.ID)
|
||||
}
|
||||
if llm.IsReservedOpenAIChatRequestField(key) {
|
||||
return domain.Backend{}, fmt.Errorf(
|
||||
"backend %q extra parameter %q collides with a reserved request field",
|
||||
definition.ID,
|
||||
key,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
|
||||
if err != nil {
|
||||
return domain.Backend{}, fmt.Errorf("backend %q extra parameters: %w", definition.ID, err)
|
||||
}
|
||||
definition.ExtraParams = extraParams
|
||||
return definition, nil
|
||||
}
|
||||
|
||||
func validateEndpoint(endpoint string) error {
|
||||
if endpoint == "" {
|
||||
return errors.New("must not be blank")
|
||||
}
|
||||
if strings.Contains(endpoint, "#") {
|
||||
return errors.New("must not contain a fragment")
|
||||
}
|
||||
|
||||
parsed, err := url.Parse(endpoint)
|
||||
if err != nil {
|
||||
return fmt.Errorf("must be a valid URL: %w", err)
|
||||
}
|
||||
scheme := strings.ToLower(parsed.Scheme)
|
||||
if scheme != "http" && scheme != "https" {
|
||||
return errors.New("must use http or https")
|
||||
}
|
||||
if !parsed.IsAbs() || parsed.Hostname() == "" {
|
||||
return errors.New("must be absolute and include a host")
|
||||
}
|
||||
if parsed.User != nil {
|
||||
return errors.New("must not contain user information")
|
||||
}
|
||||
if parsed.RawQuery != "" || parsed.ForceQuery {
|
||||
return errors.New("must not contain a query string")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
225
internal/backend/registry_test.go
Normal file
225
internal/backend/registry_test.go
Normal file
@@ -0,0 +1,225 @@
|
||||
package backend_test
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
const validEndpoint = "https://backend.example/v1"
|
||||
|
||||
func TestRegistryIncludesExactOpenRouterDefinition(t *testing.T) {
|
||||
registry, err := backend.NewRegistry(nil)
|
||||
if err != nil {
|
||||
t.Fatalf("construct registry: %v", err)
|
||||
}
|
||||
|
||||
definition, err := registry.GetBackend(backend.OpenRouterID)
|
||||
if err != nil {
|
||||
t.Fatalf("look up OpenRouter: %v", err)
|
||||
}
|
||||
if definition.ID != "openrouter" ||
|
||||
definition.Endpoint != "https://openrouter.ai/api/v1" ||
|
||||
definition.APIKeyEnv != "OPENROUTER_API_KEY" ||
|
||||
definition.ExtraParams != nil {
|
||||
t.Fatalf("unexpected OpenRouter definition: %#v", definition)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryNormalizesUniqueAdditionsAndIsolatesMutations(t *testing.T) {
|
||||
nested := map[string]int{"limit": 2}
|
||||
extraParams := map[string]any{
|
||||
"count": int64(7),
|
||||
"nested": nested,
|
||||
}
|
||||
registry, err := backend.NewRegistry([]domain.Backend{
|
||||
{
|
||||
ID: " custom ",
|
||||
Endpoint: " https://custom.example/openai/v1 ",
|
||||
APIKeyEnv: " CUSTOM_API_KEY ",
|
||||
ExtraParams: extraParams,
|
||||
},
|
||||
{
|
||||
ID: "Custom",
|
||||
Endpoint: validEndpoint,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("construct registry: %v", err)
|
||||
}
|
||||
|
||||
nested["limit"] = 99
|
||||
extraParams["added"] = true
|
||||
|
||||
got, err := registry.GetBackend("custom")
|
||||
if err != nil {
|
||||
t.Fatalf("look up custom backend: %v", err)
|
||||
}
|
||||
if got.ID != "custom" ||
|
||||
got.Endpoint != "https://custom.example/openai/v1" ||
|
||||
got.APIKeyEnv != "CUSTOM_API_KEY" {
|
||||
t.Fatalf("unexpected normalized definition: %#v", got)
|
||||
}
|
||||
if count, ok := got.ExtraParams["count"].(int64); !ok || count != 7 {
|
||||
t.Fatalf("integer type or value changed: %#v", got.ExtraParams["count"])
|
||||
}
|
||||
gotNested, ok := got.ExtraParams["nested"].(map[string]int)
|
||||
if !ok || gotNested["limit"] != 2 {
|
||||
t.Fatalf("container type or value changed: %#v", got.ExtraParams["nested"])
|
||||
}
|
||||
if _, exists := got.ExtraParams["added"]; exists {
|
||||
t.Fatalf("registry retained caller map: %#v", got.ExtraParams)
|
||||
}
|
||||
|
||||
gotNested["limit"] = 100
|
||||
got.ExtraParams["added"] = true
|
||||
again, err := registry.GetBackend("custom")
|
||||
if err != nil {
|
||||
t.Fatalf("look up custom backend again: %v", err)
|
||||
}
|
||||
if again.ExtraParams["nested"].(map[string]int)["limit"] != 2 {
|
||||
t.Fatalf("lookup exposed registry nested map: %#v", again.ExtraParams)
|
||||
}
|
||||
if _, exists := again.ExtraParams["added"]; exists {
|
||||
t.Fatalf("lookup exposed registry map: %#v", again.ExtraParams)
|
||||
}
|
||||
|
||||
if _, err := registry.GetBackend("Custom"); err != nil {
|
||||
t.Fatalf("backend IDs should be case-sensitive: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRegistryRejectsDuplicateIDs(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
additions []domain.Backend
|
||||
wantID string
|
||||
}{
|
||||
{
|
||||
name: "built-in collision after normalization",
|
||||
additions: []domain.Backend{{
|
||||
ID: " openrouter ",
|
||||
}},
|
||||
wantID: "openrouter",
|
||||
},
|
||||
{
|
||||
name: "consumer collision after normalization",
|
||||
additions: []domain.Backend{
|
||||
{ID: "custom", Endpoint: validEndpoint},
|
||||
{ID: " custom ", Endpoint: "https://other.example/v1"},
|
||||
},
|
||||
wantID: "custom",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := backend.NewRegistry(tc.additions)
|
||||
if err == nil {
|
||||
t.Fatal("expected duplicate ID error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), tc.wantID) {
|
||||
t.Fatalf("expected error to identify %q, got %v", tc.wantID, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRegistryValidatesIDs(t *testing.T) {
|
||||
for _, id := range []string{"", " \t\n "} {
|
||||
t.Run(id, func(t *testing.T) {
|
||||
_, err := backend.NewRegistry([]domain.Backend{{
|
||||
ID: id,
|
||||
Endpoint: validEndpoint,
|
||||
}})
|
||||
if err == nil {
|
||||
t.Fatal("expected blank ID error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRegistryValidatesEndpoints(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
endpoint string
|
||||
}{
|
||||
{name: "blank", endpoint: ""},
|
||||
{name: "relative", endpoint: "/v1"},
|
||||
{name: "missing host", endpoint: "https:///v1"},
|
||||
{name: "unsupported scheme", endpoint: "ftp://backend.example/v1"},
|
||||
{name: "user information", endpoint: "https://user@backend.example/v1"},
|
||||
{name: "query", endpoint: "https://backend.example/v1?mode=chat"},
|
||||
{name: "empty query", endpoint: "https://backend.example/v1?"},
|
||||
{name: "fragment", endpoint: "https://backend.example/v1#chat"},
|
||||
{name: "empty fragment", endpoint: "https://backend.example/v1#"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := backend.NewRegistry([]domain.Backend{{
|
||||
ID: "custom",
|
||||
Endpoint: tc.endpoint,
|
||||
}})
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid endpoint error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRegistryValidatesEnvironmentVariableNames(t *testing.T) {
|
||||
for _, name := range []string{"1API_KEY", "API-KEY", "API KEY", "ÅPI_KEY"} {
|
||||
t.Run(name, func(t *testing.T) {
|
||||
_, err := backend.NewRegistry([]domain.Backend{{
|
||||
ID: "custom",
|
||||
Endpoint: validEndpoint,
|
||||
APIKeyEnv: name,
|
||||
}})
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid environment-variable name error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewRegistryRejectsInvalidAndReservedExtraParameters(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
extraParams map[string]any
|
||||
}{
|
||||
{name: "unsupported value", extraParams: map[string]any{"value": make(chan int)}},
|
||||
{name: "reserved key", extraParams: map[string]any{"model": "override"}},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := backend.NewRegistry([]domain.Backend{{
|
||||
ID: "custom",
|
||||
Endpoint: validEndpoint,
|
||||
ExtraParams: tc.extraParams,
|
||||
}})
|
||||
if err == nil {
|
||||
t.Fatal("expected invalid extra parameters error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestRegistryLookupReportsNotFound(t *testing.T) {
|
||||
registry, err := backend.NewRegistry(nil)
|
||||
if err != nil {
|
||||
t.Fatalf("construct registry: %v", err)
|
||||
}
|
||||
|
||||
_, err = registry.GetBackend("missing")
|
||||
if !errors.Is(err, backend.ErrBackendNotFound) {
|
||||
t.Fatalf("expected ErrBackendNotFound, got %v", err)
|
||||
}
|
||||
if !strings.Contains(err.Error(), "missing") {
|
||||
t.Fatalf("expected error to identify backend, got %v", err)
|
||||
}
|
||||
}
|
||||
@@ -63,12 +63,12 @@ type RunRequest struct {
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
ProfileID string
|
||||
SessionID string
|
||||
APIKey string `json:"-" yaml:"-"`
|
||||
Inputs map[string]ArtifactRef
|
||||
Vars map[string]string
|
||||
Execution *ExecutionTargetOverride
|
||||
Validation *OutputContract
|
||||
Metadata map[string]string
|
||||
}
|
||||
|
||||
// RunResult represents the complete result of a prompt execution run.
|
||||
@@ -80,8 +80,10 @@ type RunResult struct {
|
||||
PromptID string
|
||||
PromptVersion string
|
||||
PromptHash string
|
||||
SessionID string
|
||||
RenderedPromptHash string
|
||||
SelectedProfileID string
|
||||
SelectedBackendID string
|
||||
ModelName string
|
||||
Endpoint string
|
||||
EffectiveModelParams ExecutionTarget
|
||||
@@ -99,6 +101,7 @@ type PreparedRun struct {
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
PromptHash string `json:"prompt_hash,omitempty"`
|
||||
SelectedProfileID string `json:"selected_profile_id"`
|
||||
SelectedBackendID string `json:"selected_backend_id,omitempty"`
|
||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||
TargetPresence ExecutionTargetPresence `json:"-"`
|
||||
OutputContract OutputContract `json:"output_contract"`
|
||||
@@ -158,9 +161,18 @@ type PromptMessageTemplate struct {
|
||||
CacheControl *CacheControl `yaml:"cache_control,omitempty" json:"cache_control,omitempty"`
|
||||
}
|
||||
|
||||
// Backend describes reusable OpenAI-compatible connection defaults.
|
||||
type Backend struct {
|
||||
ID string
|
||||
Endpoint string
|
||||
APIKeyEnv string
|
||||
ExtraParams map[string]any
|
||||
}
|
||||
|
||||
// ExecutionProfile describes how and where to execute a model.
|
||||
type ExecutionProfile struct {
|
||||
ID string `yaml:"id"`
|
||||
BackendID string `yaml:"backend"`
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
Model string `yaml:"model"`
|
||||
Temperature float64 `yaml:"temperature"`
|
||||
@@ -183,7 +195,7 @@ type ExecutionTargetOverride struct {
|
||||
TopP *float64 `json:"top_p,omitempty"`
|
||||
TimeoutSeconds *int `json:"timeout_seconds,omitempty"`
|
||||
ServiceTier string `json:"service_tier,omitempty"`
|
||||
ReasoningEffort string `json:"reasoning_effort,omitempty"`
|
||||
ReasoningEffort *string `json:"reasoning_effort,omitempty"`
|
||||
APIKeyEnv string `json:"api_key_env,omitempty"`
|
||||
ExtraParams map[string]any `json:"extra_params,omitempty"`
|
||||
}
|
||||
@@ -199,6 +211,7 @@ type ExecutionTargetPresence struct {
|
||||
|
||||
// ExecutionTarget represents effective model runtime settings for a run.
|
||||
type ExecutionTarget struct {
|
||||
BackendID string `yaml:"backend" json:"backend_id,omitempty"`
|
||||
Endpoint string `yaml:"endpoint" json:"endpoint"`
|
||||
Model string `yaml:"model" json:"model"`
|
||||
Temperature float64 `yaml:"temperature" json:"temperature"`
|
||||
|
||||
19
internal/domain/session.go
Normal file
19
internal/domain/session.go
Normal file
@@ -0,0 +1,19 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// NormalizeSessionID applies the shared session identifier rule.
|
||||
func NormalizeSessionID(raw string) (string, error) {
|
||||
normalized := strings.TrimSpace(raw)
|
||||
if normalized == "" {
|
||||
return "", nil
|
||||
}
|
||||
if length := utf8.RuneCountInString(normalized); length > SessionIDMaxLength {
|
||||
return "", fmt.Errorf("session_id length %d exceeds maximum %d", length, SessionIDMaxLength)
|
||||
}
|
||||
return normalized, nil
|
||||
}
|
||||
57
internal/domain/session_test.go
Normal file
57
internal/domain/session_test.go
Normal file
@@ -0,0 +1,57 @@
|
||||
package domain
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestNormalizeSessionID(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
raw string
|
||||
want string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "trims surrounding Unicode whitespace",
|
||||
raw: "\u2003 session-123 \u2003",
|
||||
want: "session-123",
|
||||
},
|
||||
{
|
||||
name: "blank input is omitted",
|
||||
raw: " \t\u2003 ",
|
||||
want: "",
|
||||
},
|
||||
{
|
||||
name: "maximum Unicode length is accepted",
|
||||
raw: strings.Repeat("界", SessionIDMaxLength),
|
||||
want: strings.Repeat("界", SessionIDMaxLength),
|
||||
},
|
||||
{
|
||||
name: "one Unicode code point over maximum is rejected",
|
||||
raw: strings.Repeat("界", SessionIDMaxLength+1),
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := NormalizeSessionID(tt.raw)
|
||||
if tt.wantErr {
|
||||
if err == nil {
|
||||
t.Fatal("expected normalization error")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "exceeds maximum") {
|
||||
t.Fatalf("expected useful length diagnostic, got %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("normalize session id: %v", err)
|
||||
}
|
||||
if got != tt.want {
|
||||
t.Fatalf("normalized session id = %q, want %q", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,25 +1,30 @@
|
||||
package promptkit
|
||||
// Package jsonvalue validates and defensively copies JSON-compatible value
|
||||
// trees used by public configuration and request boundaries.
|
||||
package jsonvalue
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
const maxSafeJSONInteger = 1<<53 - 1
|
||||
|
||||
type jsonVisit struct {
|
||||
type visit struct {
|
||||
typ reflect.Type
|
||||
ptr uintptr
|
||||
}
|
||||
|
||||
func copyPublicJSONMap(src map[string]any) (map[string]any, error) {
|
||||
// CopyMap validates and deeply copies an extra-parameter map while preserving
|
||||
// compatible concrete map, slice, array, scalar, and number types.
|
||||
func CopyMap(src map[string]any) (map[string]any, error) {
|
||||
if src == nil {
|
||||
return nil, nil
|
||||
}
|
||||
copied, err := copyPublicJSONValue(reflect.ValueOf(src), "extra_params", make(map[jsonVisit]struct{}))
|
||||
copied, err := copyValue(reflect.ValueOf(src), "extra_params", make(map[visit]struct{}))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -30,7 +35,7 @@ func copyPublicJSONMap(src map[string]any) (map[string]any, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func copyPublicJSONValue(value reflect.Value, path string, seen map[jsonVisit]struct{}) (any, error) {
|
||||
func copyValue(value reflect.Value, path string, seen map[visit]struct{}) (any, error) {
|
||||
if !value.IsValid() {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -38,12 +43,15 @@ func copyPublicJSONValue(value reflect.Value, path string, seen map[jsonVisit]st
|
||||
if value.IsNil() {
|
||||
return nil, nil
|
||||
}
|
||||
return copyPublicJSONValue(value.Elem(), path, seen)
|
||||
return copyValue(value.Elem(), path, seen)
|
||||
}
|
||||
if !value.CanInterface() {
|
||||
return nil, fmt.Errorf("%s: value cannot be copied", path)
|
||||
}
|
||||
if number, ok := value.Interface().(json.Number); ok {
|
||||
if _, err := json.Marshal(number); err != nil {
|
||||
return nil, fmt.Errorf("%s: invalid JSON number", path)
|
||||
}
|
||||
f, err := strconv.ParseFloat(number.String(), 64)
|
||||
if err != nil || math.IsNaN(f) || math.IsInf(f, 0) {
|
||||
return nil, fmt.Errorf("%s: invalid JSON number", path)
|
||||
@@ -65,8 +73,8 @@ func copyPublicJSONValue(value reflect.Value, path string, seen map[jsonVisit]st
|
||||
}
|
||||
return value.Interface(), nil
|
||||
case reflect.Float32, reflect.Float64:
|
||||
f := value.Convert(reflect.TypeOf(float64(0))).Float()
|
||||
if math.IsNaN(f) || math.IsInf(f, 0) {
|
||||
number := value.Convert(reflect.TypeOf(float64(0))).Float()
|
||||
if math.IsNaN(number) || math.IsInf(number, 0) {
|
||||
return nil, fmt.Errorf("%s: floating-point value must be finite", path)
|
||||
}
|
||||
return value.Interface(), nil
|
||||
@@ -74,28 +82,28 @@ func copyPublicJSONValue(value reflect.Value, path string, seen map[jsonVisit]st
|
||||
if value.IsNil() {
|
||||
return nil, nil
|
||||
}
|
||||
visit := jsonVisit{typ: value.Type(), ptr: value.Pointer()}
|
||||
if _, ok := seen[visit]; ok {
|
||||
current := visit{typ: value.Type(), ptr: value.Pointer()}
|
||||
if _, ok := seen[current]; ok {
|
||||
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
|
||||
}
|
||||
seen[visit] = struct{}{}
|
||||
defer delete(seen, visit)
|
||||
return copyPublicJSONValue(value.Elem(), path, seen)
|
||||
seen[current] = struct{}{}
|
||||
defer delete(seen, current)
|
||||
return copyValue(value.Elem(), path, seen)
|
||||
case reflect.Map:
|
||||
return copyPublicJSONMapValue(value, path, seen)
|
||||
return copyMapValue(value, path, seen)
|
||||
case reflect.Slice:
|
||||
if value.IsNil() {
|
||||
return nil, nil
|
||||
}
|
||||
return copyPublicJSONSequenceValue(value, path, seen)
|
||||
return copySequenceValue(value, path, seen)
|
||||
case reflect.Array:
|
||||
return copyPublicJSONSequenceValue(value, path, seen)
|
||||
return copySequenceValue(value, path, seen)
|
||||
default:
|
||||
return nil, fmt.Errorf("%s: unsupported JSON value type %s", path, value.Type())
|
||||
}
|
||||
}
|
||||
|
||||
func copyPublicJSONMapValue(value reflect.Value, path string, seen map[jsonVisit]struct{}) (any, error) {
|
||||
func copyMapValue(value reflect.Value, path string, seen map[visit]struct{}) (any, error) {
|
||||
if value.IsNil() {
|
||||
return nil, nil
|
||||
}
|
||||
@@ -103,37 +111,43 @@ func copyPublicJSONMapValue(value reflect.Value, path string, seen map[jsonVisit
|
||||
return nil, fmt.Errorf("%s: map key type %s is not supported", path, value.Type().Key())
|
||||
}
|
||||
|
||||
visit := jsonVisit{typ: value.Type(), ptr: value.Pointer()}
|
||||
if _, ok := seen[visit]; ok {
|
||||
current := visit{typ: value.Type(), ptr: value.Pointer()}
|
||||
if _, ok := seen[current]; ok {
|
||||
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
|
||||
}
|
||||
seen[visit] = struct{}{}
|
||||
defer delete(seen, visit)
|
||||
seen[current] = struct{}{}
|
||||
defer delete(seen, current)
|
||||
|
||||
keys := value.MapKeys()
|
||||
sort.Slice(keys, func(i, j int) bool {
|
||||
return keys[i].String() < keys[j].String()
|
||||
})
|
||||
|
||||
type entry struct {
|
||||
key reflect.Value
|
||||
name string
|
||||
value any
|
||||
}
|
||||
entries := make([]entry, 0, value.Len())
|
||||
entries := make([]entry, 0, len(keys))
|
||||
preserveType := true
|
||||
elemType := value.Type().Elem()
|
||||
iter := value.MapRange()
|
||||
for iter.Next() {
|
||||
key := iter.Key()
|
||||
elementType := value.Type().Elem()
|
||||
for _, key := range keys {
|
||||
name := key.String()
|
||||
copied, err := copyPublicJSONValue(iter.Value(), path+"."+name, seen)
|
||||
if name == "" {
|
||||
return nil, fmt.Errorf("%s: map key must not be empty", path)
|
||||
}
|
||||
copied, err := copyValue(value.MapIndex(key), path+"."+name, seen)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
entries = append(entries, entry{key: key, name: name, value: copied})
|
||||
if copied == nil {
|
||||
if !canAssignNil(elemType) {
|
||||
if !canAssignNil(elementType) {
|
||||
preserveType = false
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !reflect.TypeOf(copied).AssignableTo(elemType) {
|
||||
if !reflect.TypeOf(copied).AssignableTo(elementType) {
|
||||
preserveType = false
|
||||
}
|
||||
}
|
||||
@@ -142,7 +156,7 @@ func copyPublicJSONMapValue(value reflect.Value, path string, seen map[jsonVisit
|
||||
out := reflect.MakeMapWithSize(value.Type(), len(entries))
|
||||
for _, entry := range entries {
|
||||
if entry.value == nil {
|
||||
out.SetMapIndex(entry.key, reflect.Zero(elemType))
|
||||
out.SetMapIndex(entry.key, reflect.Zero(elementType))
|
||||
continue
|
||||
}
|
||||
out.SetMapIndex(entry.key, reflect.ValueOf(entry.value))
|
||||
@@ -157,33 +171,33 @@ func copyPublicJSONMapValue(value reflect.Value, path string, seen map[jsonVisit
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func copyPublicJSONSequenceValue(value reflect.Value, path string, seen map[jsonVisit]struct{}) (any, error) {
|
||||
var visit jsonVisit
|
||||
func copySequenceValue(value reflect.Value, path string, seen map[visit]struct{}) (any, error) {
|
||||
var current visit
|
||||
if value.Kind() == reflect.Slice {
|
||||
visit = jsonVisit{typ: value.Type(), ptr: value.Pointer()}
|
||||
if _, ok := seen[visit]; ok {
|
||||
current = visit{typ: value.Type(), ptr: value.Pointer()}
|
||||
if _, ok := seen[current]; ok {
|
||||
return nil, fmt.Errorf("%s: cyclic value is not supported", path)
|
||||
}
|
||||
seen[visit] = struct{}{}
|
||||
defer delete(seen, visit)
|
||||
seen[current] = struct{}{}
|
||||
defer delete(seen, current)
|
||||
}
|
||||
|
||||
values := make([]any, value.Len())
|
||||
preserveType := true
|
||||
elemType := value.Type().Elem()
|
||||
elementType := value.Type().Elem()
|
||||
for i := 0; i < value.Len(); i++ {
|
||||
copied, err := copyPublicJSONValue(value.Index(i), fmt.Sprintf("%s[%d]", path, i), seen)
|
||||
copied, err := copyValue(value.Index(i), fmt.Sprintf("%s[%d]", path, i), seen)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
values[i] = copied
|
||||
if copied == nil {
|
||||
if !canAssignNil(elemType) {
|
||||
if !canAssignNil(elementType) {
|
||||
preserveType = false
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !reflect.TypeOf(copied).AssignableTo(elemType) {
|
||||
if !reflect.TypeOf(copied).AssignableTo(elementType) {
|
||||
preserveType = false
|
||||
}
|
||||
}
|
||||
@@ -195,7 +209,7 @@ func copyPublicJSONSequenceValue(value reflect.Value, path string, seen map[json
|
||||
}
|
||||
for i, copied := range values {
|
||||
if copied == nil {
|
||||
out.Index(i).Set(reflect.Zero(elemType))
|
||||
out.Index(i).Set(reflect.Zero(elementType))
|
||||
continue
|
||||
}
|
||||
out.Index(i).Set(reflect.ValueOf(copied))
|
||||
97
internal/jsonvalue/jsonvalue_test.go
Normal file
97
internal/jsonvalue/jsonvalue_test.go
Normal file
@@ -0,0 +1,97 @@
|
||||
package jsonvalue_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"math"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||
)
|
||||
|
||||
func TestCopyMapPreservesTypesAndIsolatesMutations(t *testing.T) {
|
||||
nested := map[string]int{"limit": 2}
|
||||
sequence := []string{"one", "two"}
|
||||
input := map[string]any{
|
||||
"count": int64(7),
|
||||
"number": json.Number("-1.25e+2"),
|
||||
"nested": nested,
|
||||
"sequence": sequence,
|
||||
}
|
||||
|
||||
copied, err := jsonvalue.CopyMap(input)
|
||||
if err != nil {
|
||||
t.Fatalf("copy map: %v", err)
|
||||
}
|
||||
nested["limit"] = 99
|
||||
sequence[0] = "changed"
|
||||
input["added"] = true
|
||||
|
||||
if got, ok := copied["count"].(int64); !ok || got != 7 {
|
||||
t.Fatalf("integer type or value changed: %#v", copied["count"])
|
||||
}
|
||||
if got, ok := copied["number"].(json.Number); !ok || got != "-1.25e+2" {
|
||||
t.Fatalf("JSON number type or value changed: %#v", copied["number"])
|
||||
}
|
||||
if got := copied["nested"].(map[string]int)["limit"]; got != 2 {
|
||||
t.Fatalf("nested map was not isolated: %d", got)
|
||||
}
|
||||
if got := copied["sequence"].([]string)[0]; got != "one" {
|
||||
t.Fatalf("sequence was not isolated: %q", got)
|
||||
}
|
||||
if _, ok := copied["added"]; ok {
|
||||
t.Fatalf("top-level map was not isolated: %#v", copied)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyMapRejectsInvalidValues(t *testing.T) {
|
||||
cyclicMap := map[string]any{}
|
||||
cyclicMap["self"] = cyclicMap
|
||||
cyclicSlice := []any{nil}
|
||||
cyclicSlice[0] = cyclicSlice
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
value any
|
||||
}{
|
||||
{name: "empty nested key", value: map[string]int{"": 1}},
|
||||
{name: "non-string map key", value: map[int]string{1: "one"}},
|
||||
{name: "unsupported value", value: make(chan int)},
|
||||
{name: "cyclic map", value: cyclicMap},
|
||||
{name: "cyclic slice", value: cyclicSlice},
|
||||
{name: "NaN", value: math.NaN()},
|
||||
{name: "positive infinity", value: math.Inf(1)},
|
||||
{name: "unsafe signed integer", value: int64(1 << 53)},
|
||||
{name: "unsafe unsigned integer", value: uint64(1 << 53)},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if _, err := jsonvalue.CopyMap(map[string]any{"value": tc.value}); err == nil {
|
||||
t.Fatal("expected validation error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestCopyMapValidatesJSONNumberSyntaxAndRange(t *testing.T) {
|
||||
for _, number := range []json.Number{"0", "-1", "1.25", "-1.25e+2"} {
|
||||
t.Run("valid "+number.String(), func(t *testing.T) {
|
||||
got, err := jsonvalue.CopyMap(map[string]any{"value": number})
|
||||
if err != nil {
|
||||
t.Fatalf("copy valid JSON number: %v", err)
|
||||
}
|
||||
if !reflect.DeepEqual(got["value"], number) {
|
||||
t.Fatalf("JSON number changed: got %#v want %#v", got["value"], number)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
for _, number := range []json.Number{"", "01", "+1", "1.", ".1", "1e9999", "not-a-number"} {
|
||||
t.Run("invalid "+number.String(), func(t *testing.T) {
|
||||
if _, err := jsonvalue.CopyMap(map[string]any{"value": number}); err == nil {
|
||||
t.Fatal("expected invalid JSON number error")
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -12,7 +12,6 @@ import (
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
@@ -177,12 +176,11 @@ func openAIChatRequestFromGenerateRequest(req domain.GenerateRequest, defaultMod
|
||||
wireReq := openAIChatRequest{
|
||||
Model: model,
|
||||
}
|
||||
if sessionID := strings.TrimSpace(req.Prompt.SessionID); sessionID != "" {
|
||||
if n := utf8.RuneCountInString(sessionID); n > domain.SessionIDMaxLength {
|
||||
return openAIChatRequest{}, fmt.Errorf("session_id length %d exceeds maximum %d", n, domain.SessionIDMaxLength)
|
||||
}
|
||||
wireReq.SessionID = sessionID
|
||||
sessionID, err := domain.NormalizeSessionID(req.Prompt.SessionID)
|
||||
if err != nil {
|
||||
return openAIChatRequest{}, err
|
||||
}
|
||||
wireReq.SessionID = sessionID
|
||||
|
||||
wireReq.Messages = make([]openAIChatRequestMessage, 0, len(req.Prompt.Messages))
|
||||
for _, msg := range req.Prompt.Messages {
|
||||
@@ -262,7 +260,7 @@ func openAIChatRequestPayload(req openAIChatRequest) (map[string]any, error) {
|
||||
if key == "" {
|
||||
return nil, errors.New("extra_params key must not be empty")
|
||||
}
|
||||
if _, reserved := reservedOpenAIChatRequestFields[key]; reserved {
|
||||
if IsReservedOpenAIChatRequestField(key) {
|
||||
return nil, fmt.Errorf("extra_params key %q collides with reserved request field", key)
|
||||
}
|
||||
if _, err := json.Marshal(value); err != nil {
|
||||
@@ -274,16 +272,23 @@ func openAIChatRequestPayload(req openAIChatRequest) (map[string]any, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
var reservedOpenAIChatRequestFields = map[string]struct{}{
|
||||
"model": {},
|
||||
"session_id": {},
|
||||
"messages": {},
|
||||
"temperature": {},
|
||||
"max_tokens": {},
|
||||
"top_p": {},
|
||||
"service_tier": {},
|
||||
"reasoning_effort": {},
|
||||
"response_format": {},
|
||||
// IsReservedOpenAIChatRequestField reports whether name is owned by the
|
||||
// standard OpenAI-compatible chat request rather than extra parameters.
|
||||
func IsReservedOpenAIChatRequestField(name string) bool {
|
||||
switch name {
|
||||
case "model",
|
||||
"session_id",
|
||||
"messages",
|
||||
"temperature",
|
||||
"max_tokens",
|
||||
"top_p",
|
||||
"service_tier",
|
||||
"reasoning_effort",
|
||||
"response_format":
|
||||
return true
|
||||
default:
|
||||
return false
|
||||
}
|
||||
}
|
||||
|
||||
type openAIChatRequestMessage struct {
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
id: aion-2
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: aion-labs/aion-2.0
|
||||
temperature: 0.72
|
||||
reasoning_effort: high
|
||||
top_p: 0.95
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: claude-fable-latest
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "~anthropic/claude-fable-latest"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 600
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: claude-haiku-latest
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "~anthropic/claude-haiku-latest"
|
||||
reasoning_effort: medium
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: claude-opus-latest
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "~anthropic/claude-opus-latest"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: claude-sonnet-latest
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "~anthropic/claude-sonnet-latest"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: deepseek-3-2
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: deepseek/deepseek-v3.2
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: deepseek-4-flash
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: deepseek/deepseek-v4-flash
|
||||
#reasoning_effort: medium
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: deepseek-4-pro
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: deepseek/deepseek-v4-pro
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
id: gemini-2-flash-lite
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "google/gemini-2.5-flash-lite"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
id: gemini-2-flash
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "google/gemini-2.5-flash"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
id: gemini-2-pro
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "google/gemini-2.5-pro"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
id: gemini-3-flash-lite
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "google/gemini-3.1-flash-lite"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
id: gemini-flash-latest
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "~google/gemini-flash-latest"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
id: gemini-pro-latest
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "~google/gemini-pro-latest"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
id: gemma-4-31b
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: google/gemma-4-31b-it:exacto
|
||||
temperature: 0.15
|
||||
reasoning_effort: high
|
||||
top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
id: minimax-m2
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: minimax/minimax-m2.5
|
||||
temperature: 0.5
|
||||
reasoning_effort: high
|
||||
top_p: 0.95
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
id: minimax-m3
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: minimax/minimax-m3
|
||||
#temperature: 0.5
|
||||
reasoning_effort: high
|
||||
#top_p: 0.95
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: mistral-large-2512
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: mistralai/mistral-large-2512
|
||||
temperature: 0.15
|
||||
top_p: 0.98
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
id: mistral-medium-3-5
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: mistralai/mistral-medium-3-5
|
||||
temperature: 0.15
|
||||
reasoning_effort: high
|
||||
top_p: 0.98
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: mistral-small-3
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: mistralai/mistral-small-3.2-24b-instruct
|
||||
temperature: 0.05
|
||||
top_p: 1.0
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
id: mistral-small-4
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: mistralai/mistral-small-2603
|
||||
temperature: 0.1
|
||||
reasoning_effort: high
|
||||
top_p: 0.98
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: nemotron-3-ultra
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: nvidia/nemotron-3-ultra-550b-a55b
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: gpt-5-mini
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "openai/gpt-5.4-mini"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: gpt-5-nano
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "openai/gpt-5.4-nano"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
"gopkg.in/yaml.v3"
|
||||
@@ -28,6 +29,12 @@ func TestBuiltInProfilesValidateThroughRepository(t *testing.T) {
|
||||
if p.ID != id {
|
||||
t.Fatalf("expected profile id %q, got %q", id, p.ID)
|
||||
}
|
||||
if p.BackendID != backend.OpenRouterID {
|
||||
t.Fatalf("expected profile %q to select %q, got %q", id, backend.OpenRouterID, p.BackendID)
|
||||
}
|
||||
if p.Endpoint != "" || p.APIKeyEnv != "" {
|
||||
t.Fatalf("expected profile %q to inherit backend connection settings, got endpoint=%q api_key_env=%q", id, p.Endpoint, p.APIKeyEnv)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -60,6 +67,15 @@ func loadBuiltInProfileIDs(t *testing.T) map[string]string {
|
||||
if _, ok := raw["api_key"]; ok {
|
||||
t.Fatalf("built-in profile %s contains raw api_key", name)
|
||||
}
|
||||
if raw["backend"] != backend.OpenRouterID {
|
||||
t.Fatalf("built-in profile %s does not select %q", name, backend.OpenRouterID)
|
||||
}
|
||||
if _, ok := raw["endpoint"]; ok {
|
||||
t.Fatalf("built-in profile %s repeats endpoint", name)
|
||||
}
|
||||
if _, ok := raw["api_key_env"]; ok {
|
||||
t.Fatalf("built-in profile %s repeats api_key_env", name)
|
||||
}
|
||||
id, ok := raw["id"].(string)
|
||||
if !ok || strings.TrimSpace(id) == "" {
|
||||
t.Fatalf("built-in profile %s has missing id", name)
|
||||
|
||||
@@ -121,6 +121,7 @@ func loadProfile(ctx context.Context, fsys fs.FS, root string, id string) (*doma
|
||||
if prof.ID != id {
|
||||
continue
|
||||
}
|
||||
prof.BackendID = strings.TrimSpace(prof.BackendID)
|
||||
if err := validateProfile(&prof); err != nil {
|
||||
if errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
||||
return nil, fmt.Errorf("%w: %s", err, relPath)
|
||||
@@ -189,8 +190,8 @@ func validateProfile(p *domain.ExecutionProfile) error {
|
||||
if strings.TrimSpace(p.ID) == "" {
|
||||
return errors.New("id is required")
|
||||
}
|
||||
if strings.TrimSpace(p.Endpoint) == "" {
|
||||
return errors.New("endpoint is required")
|
||||
if strings.TrimSpace(p.BackendID) == "" && strings.TrimSpace(p.Endpoint) == "" {
|
||||
return errors.New("backend or endpoint is required")
|
||||
}
|
||||
if strings.TrimSpace(p.Model) == "" {
|
||||
return errors.New("model is required")
|
||||
|
||||
@@ -52,6 +52,43 @@ func TestFilesystemRepository_GetProfile(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("backend and endpoint connection matrix", func(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
connection string
|
||||
wantBackend string
|
||||
wantEndpoint string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "backend only", connection: "backend: ' openrouter '", wantBackend: "openrouter"},
|
||||
{name: "endpoint only", connection: "endpoint: http://localhost:8000/v1", wantEndpoint: "http://localhost:8000/v1"},
|
||||
{name: "both", connection: "backend: openrouter\nendpoint: http://localhost:8000/v1", wantBackend: "openrouter", wantEndpoint: "http://localhost:8000/v1"},
|
||||
{name: "neither", wantErr: true},
|
||||
{name: "blank backend", connection: "backend: ' '", wantErr: true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
id := "connection-" + strings.ReplaceAll(tt.name, " ", "-")
|
||||
writeProfileTestFile(t, filepath.Join(tmpDir, id+".yaml"), "id: "+id+"\nmodel: model\n"+tt.connection+"\n")
|
||||
|
||||
p, err := repo.GetProfile(ctx, id)
|
||||
if tt.wantErr {
|
||||
if !errors.Is(err, ErrInvalidProfile) {
|
||||
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("expected profile to load, got %v", err)
|
||||
}
|
||||
if p.BackendID != tt.wantBackend || p.Endpoint != tt.wantEndpoint {
|
||||
t.Fatalf("unexpected connection values: backend=%q endpoint=%q", p.BackendID, p.Endpoint)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("valid profile with api_key_env", func(t *testing.T) {
|
||||
p, err := repo.GetProfile(ctx, "local-secure")
|
||||
if err != nil {
|
||||
|
||||
@@ -5,10 +5,9 @@ import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"strings"
|
||||
"text/template"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
)
|
||||
|
||||
var (
|
||||
@@ -95,10 +94,6 @@ func (r *goRenderer) Render(ctx context.Context, definition *domain.PromptDefini
|
||||
}
|
||||
|
||||
func renderSessionID(raw string, funcs template.FuncMap, vars map[string]string) (string, error) {
|
||||
if strings.TrimSpace(raw) == "" {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
tmpl, err := template.New("session_id").Funcs(funcs).Option("missingkey=error").Parse(raw)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%w: session_id: %v", ErrInvalidTemplate, err)
|
||||
@@ -109,9 +104,9 @@ func renderSessionID(raw string, funcs template.FuncMap, vars map[string]string)
|
||||
return "", fmt.Errorf("%w: session_id: %w", ErrRenderFailure, err)
|
||||
}
|
||||
|
||||
sessionID := strings.TrimSpace(buf.String())
|
||||
if n := utf8.RuneCountInString(sessionID); n > domain.SessionIDMaxLength {
|
||||
return "", fmt.Errorf("%w: session_id length %d exceeds maximum %d", ErrRenderFailure, n, domain.SessionIDMaxLength)
|
||||
sessionID, err := domain.NormalizeSessionID(buf.String())
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%w: session_id: %v", ErrRenderFailure, err)
|
||||
}
|
||||
return sessionID, nil
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ type OutputRepairer interface {
|
||||
type RepairRequest struct {
|
||||
PreviousOutput string
|
||||
ValidationErrors []string
|
||||
SessionID string
|
||||
Target domain.ExecutionTarget
|
||||
StructuredOutput *domain.StructuredOutputSpec
|
||||
Attempt int
|
||||
@@ -42,23 +43,26 @@ func (r *defaultOutputRepairer) Repair(ctx context.Context, req RepairRequest) (
|
||||
errs = strings.Join(req.ValidationErrors, "\n")
|
||||
}
|
||||
|
||||
prompt := domain.RenderedPrompt{Messages: []domain.RenderedMessage{
|
||||
{
|
||||
Role: "system",
|
||||
Content: "You repair invalid JSON output. Return only corrected JSON. Do not include explanations or markdown code fences.",
|
||||
prompt := domain.RenderedPrompt{
|
||||
SessionID: req.SessionID,
|
||||
Messages: []domain.RenderedMessage{
|
||||
{
|
||||
Role: "system",
|
||||
Content: "You repair invalid JSON output. Return only corrected JSON. Do not include explanations or markdown code fences.",
|
||||
},
|
||||
{
|
||||
Role: "user",
|
||||
Content: fmt.Sprintf(
|
||||
"Repair attempt %d of %d for validation mode %s.\n\nValidation errors:\n%s\n\nPrevious output:\n%s\n\nReturn only corrected JSON.",
|
||||
req.Attempt,
|
||||
req.MaxAttempts,
|
||||
req.Mode,
|
||||
errs,
|
||||
req.PreviousOutput,
|
||||
),
|
||||
},
|
||||
},
|
||||
{
|
||||
Role: "user",
|
||||
Content: fmt.Sprintf(
|
||||
"Repair attempt %d of %d for validation mode %s.\n\nValidation errors:\n%s\n\nPrevious output:\n%s\n\nReturn only corrected JSON.",
|
||||
req.Attempt,
|
||||
req.MaxAttempts,
|
||||
req.Mode,
|
||||
errs,
|
||||
req.PreviousOutput,
|
||||
),
|
||||
},
|
||||
}}
|
||||
}
|
||||
|
||||
resp, err := r.llm.Generate(ctx, domain.GenerateRequest{
|
||||
Prompt: prompt,
|
||||
|
||||
@@ -40,6 +40,7 @@ var (
|
||||
type Runner struct {
|
||||
promptDefs promptdef.Repository
|
||||
profiles profile.Repository
|
||||
backends BackendResolver
|
||||
artifacts artifact.Reader
|
||||
renderer prompt.Renderer
|
||||
llm llm.Client
|
||||
@@ -47,20 +48,27 @@ type Runner struct {
|
||||
repairer OutputRepairer
|
||||
}
|
||||
|
||||
// BackendResolver resolves one normalized backend ID.
|
||||
type BackendResolver interface {
|
||||
GetBackend(string) (domain.Backend, error)
|
||||
}
|
||||
|
||||
func NewRunner(
|
||||
promptDefs promptdef.Repository,
|
||||
profiles profile.Repository,
|
||||
backends BackendResolver,
|
||||
artifacts artifact.Reader,
|
||||
renderer prompt.Renderer,
|
||||
llmClient llm.Client,
|
||||
validator validate.Validator,
|
||||
) *Runner {
|
||||
return NewRunnerWithRepairer(promptDefs, profiles, artifacts, renderer, llmClient, validator, nil)
|
||||
return NewRunnerWithRepairer(promptDefs, profiles, backends, artifacts, renderer, llmClient, validator, nil)
|
||||
}
|
||||
|
||||
func NewRunnerWithRepairer(
|
||||
promptDefs promptdef.Repository,
|
||||
profiles profile.Repository,
|
||||
backends BackendResolver,
|
||||
artifacts artifact.Reader,
|
||||
renderer prompt.Renderer,
|
||||
llmClient llm.Client,
|
||||
@@ -70,6 +78,7 @@ func NewRunnerWithRepairer(
|
||||
return &Runner{
|
||||
promptDefs: promptDefs,
|
||||
profiles: profiles,
|
||||
backends: backends,
|
||||
artifacts: artifacts,
|
||||
renderer: renderer,
|
||||
llm: llmClient,
|
||||
@@ -118,6 +127,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
||||
repairResp, repairErr := r.repairer.Repair(ctx, RepairRequest{
|
||||
PreviousOutput: genResp.Content,
|
||||
ValidationErrors: validationResult.Errors,
|
||||
SessionID: prepared.SessionID,
|
||||
Target: prepared.EffectiveModelParams,
|
||||
StructuredOutput: prepared.StructuredOutput,
|
||||
Attempt: attemptsUsed,
|
||||
@@ -151,8 +161,10 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
||||
PromptID: prepared.PromptID,
|
||||
PromptVersion: prepared.PromptVersion,
|
||||
PromptHash: prepared.PromptHash,
|
||||
SessionID: prepared.SessionID,
|
||||
RenderedPromptHash: prepared.RenderedPromptHash,
|
||||
SelectedProfileID: prepared.SelectedProfileID,
|
||||
SelectedBackendID: prepared.SelectedBackendID,
|
||||
ModelName: prepared.EffectiveModelParams.Model,
|
||||
Endpoint: prepared.EffectiveModelParams.Endpoint,
|
||||
EffectiveModelParams: prepared.EffectiveModelParams,
|
||||
@@ -168,6 +180,10 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
|
||||
if strings.TrimSpace(req.PromptID) == "" {
|
||||
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidRequest)
|
||||
}
|
||||
directSessionID, err := domain.NormalizeSessionID(req.SessionID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: session_id: %v", ErrInvalidRequest, err)
|
||||
}
|
||||
|
||||
start := time.Now().UTC()
|
||||
|
||||
@@ -193,7 +209,20 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
|
||||
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
||||
}
|
||||
|
||||
effectiveModel, targetPresence, err := resolveExecutionTarget(execProfile, req.Execution)
|
||||
var selectedBackend *domain.Backend
|
||||
if backendID := strings.TrimSpace(execProfile.BackendID); backendID != "" {
|
||||
execProfile.BackendID = backendID
|
||||
if r.backends == nil {
|
||||
return nil, fmt.Errorf("%w: backend %q cannot be resolved", ErrProfileLoad, backendID)
|
||||
}
|
||||
resolvedBackend, resolveErr := r.backends.GetBackend(backendID)
|
||||
if resolveErr != nil {
|
||||
return nil, fmt.Errorf("%w: backend %q: %w", ErrProfileLoad, backendID, resolveErr)
|
||||
}
|
||||
selectedBackend = &resolvedBackend
|
||||
}
|
||||
|
||||
effectiveModel, targetPresence, err := resolveExecutionTarget(selectedBackend, execProfile, req.Execution)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
||||
}
|
||||
@@ -228,10 +257,19 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
|
||||
inputHashes[name] = art.Hash
|
||||
}
|
||||
|
||||
renderedPrompt, err := r.renderer.Render(ctx, def, resolvedInputs, req.Vars)
|
||||
definitionToRender := def
|
||||
if directSessionID != "" {
|
||||
definitionCopy := *def
|
||||
definitionCopy.SessionID = ""
|
||||
definitionToRender = &definitionCopy
|
||||
}
|
||||
renderedPrompt, err := r.renderer.Render(ctx, definitionToRender, resolvedInputs, req.Vars)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrPromptRender, err)
|
||||
}
|
||||
if directSessionID != "" {
|
||||
renderedPrompt.SessionID = directSessionID
|
||||
}
|
||||
|
||||
end := time.Now().UTC()
|
||||
return &domain.PreparedRun{
|
||||
@@ -239,6 +277,7 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
|
||||
PromptVersion: def.Version,
|
||||
PromptHash: promptDefinitionHash,
|
||||
SelectedProfileID: selectedProfileID,
|
||||
SelectedBackendID: effectiveModel.BackendID,
|
||||
EffectiveModelParams: effectiveModel,
|
||||
TargetPresence: targetPresence,
|
||||
OutputContract: effectiveContract,
|
||||
@@ -338,7 +377,10 @@ func (r *Runner) shouldAttemptRepair(contract domain.OutputContract, validationR
|
||||
|
||||
func mergeExecutionTarget(base domain.ExecutionTarget, override domain.ExecutionTarget) domain.ExecutionTarget {
|
||||
out := base
|
||||
if override.Endpoint != "" {
|
||||
if strings.TrimSpace(override.BackendID) != "" {
|
||||
out.BackendID = override.BackendID
|
||||
}
|
||||
if strings.TrimSpace(override.Endpoint) != "" {
|
||||
out.Endpoint = override.Endpoint
|
||||
}
|
||||
if override.Model != "" {
|
||||
@@ -367,6 +409,7 @@ func mergeExecutionTarget(base domain.ExecutionTarget, override domain.Execution
|
||||
}
|
||||
if override.APIKeyRequired {
|
||||
out.APIKeyRequired = true
|
||||
out.APIKeyEnv = ""
|
||||
}
|
||||
if len(override.ExtraParams) > 0 {
|
||||
out.ExtraParams = copyExtraParams(override.ExtraParams)
|
||||
@@ -414,8 +457,8 @@ func mergeExecutionTargetOverride(base domain.ExecutionTarget, override domain.E
|
||||
if strings.TrimSpace(override.ServiceTier) != "" {
|
||||
out.ServiceTier = override.ServiceTier
|
||||
}
|
||||
if strings.TrimSpace(override.ReasoningEffort) != "" {
|
||||
out.ReasoningEffort = override.ReasoningEffort
|
||||
if override.ReasoningEffort != nil {
|
||||
out.ReasoningEffort = strings.TrimSpace(*override.ReasoningEffort)
|
||||
}
|
||||
if strings.TrimSpace(override.APIKeyEnv) != "" {
|
||||
out.APIKeyEnv = override.APIKeyEnv
|
||||
@@ -426,8 +469,9 @@ func mergeExecutionTargetOverride(base domain.ExecutionTarget, override domain.E
|
||||
return out, presence, nil
|
||||
}
|
||||
|
||||
func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) {
|
||||
func resolveExecutionTarget(backendValue *domain.Backend, profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) {
|
||||
out := defaults.ExecutionTargetDefault()
|
||||
out = mergeExecutionTarget(out, backendToTarget(backendValue))
|
||||
out = mergeExecutionTarget(out, executionProfileToTarget(profileValue))
|
||||
var presence domain.ExecutionTargetPresence
|
||||
if override != nil {
|
||||
@@ -461,8 +505,13 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
|
||||
if p == nil {
|
||||
return domain.ExecutionTarget{}
|
||||
}
|
||||
endpoint := p.Endpoint
|
||||
if strings.TrimSpace(endpoint) == "" {
|
||||
endpoint = ""
|
||||
}
|
||||
return domain.ExecutionTarget{
|
||||
Endpoint: p.Endpoint,
|
||||
BackendID: p.BackendID,
|
||||
Endpoint: endpoint,
|
||||
Model: p.Model,
|
||||
Temperature: p.Temperature,
|
||||
MaxTokens: p.MaxTokens,
|
||||
@@ -476,6 +525,18 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
|
||||
}
|
||||
}
|
||||
|
||||
func backendToTarget(value *domain.Backend) domain.ExecutionTarget {
|
||||
if value == nil {
|
||||
return domain.ExecutionTarget{}
|
||||
}
|
||||
return domain.ExecutionTarget{
|
||||
BackendID: value.ID,
|
||||
Endpoint: value.Endpoint,
|
||||
APIKeyEnv: value.APIKeyEnv,
|
||||
ExtraParams: copyExtraParams(value.ExtraParams),
|
||||
}
|
||||
}
|
||||
|
||||
func copyExtraParams(src map[string]any) map[string]any {
|
||||
if len(src) == 0 {
|
||||
return nil
|
||||
|
||||
@@ -34,6 +34,18 @@ type fakeExecutionProfileRepo struct {
|
||||
lastID string
|
||||
}
|
||||
|
||||
type fakeBackendResolver struct {
|
||||
backends map[string]domain.Backend
|
||||
}
|
||||
|
||||
func (f fakeBackendResolver) GetBackend(id string) (domain.Backend, error) {
|
||||
value, ok := f.backends[id]
|
||||
if !ok {
|
||||
return domain.Backend{}, errors.New("backend not found")
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func (f *fakeExecutionProfileRepo) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||
f.lastID = id
|
||||
if f.err != nil {
|
||||
@@ -167,7 +179,7 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) {
|
||||
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
||||
llmClient := &fakeLLM{forbid: true}
|
||||
|
||||
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil)
|
||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
PromptVersion: "1",
|
||||
@@ -210,6 +222,145 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerDirectSessionResolution(t *testing.T) {
|
||||
t.Run("direct value wins and changes only the rendered prompt hash", func(t *testing.T) {
|
||||
def := promptDef(domain.FormatText, domain.ValidationNone, 0)
|
||||
def.SessionID = "template-{{.template_session}}"
|
||||
promptRepo := &fakePromptRepo{def: def}
|
||||
runner := NewRunner(
|
||||
promptRepo,
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
nil,
|
||||
defaultArtifactReader(),
|
||||
prompt.NewGoRenderer(),
|
||||
&fakeLLM{forbid: true},
|
||||
nil,
|
||||
)
|
||||
req := domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
Inputs: singleInputRef(),
|
||||
Vars: map[string]string{"template_session": "from-template"},
|
||||
}
|
||||
|
||||
req.SessionID = " direct-one "
|
||||
first, err := runner.Prepare(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare first direct session: %v", err)
|
||||
}
|
||||
req.SessionID = "direct-two"
|
||||
second, err := runner.Prepare(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare second direct session: %v", err)
|
||||
}
|
||||
|
||||
if first.SessionID != "direct-one" || second.SessionID != "direct-two" {
|
||||
t.Fatalf("direct sessions were not normalized: first=%q second=%q", first.SessionID, second.SessionID)
|
||||
}
|
||||
if first.PromptHash != second.PromptHash {
|
||||
t.Fatalf("direct session changed prompt-definition hash: first=%q second=%q", first.PromptHash, second.PromptHash)
|
||||
}
|
||||
if first.RenderedPromptHash == second.RenderedPromptHash {
|
||||
t.Fatal("changing direct session did not change rendered-prompt hash")
|
||||
}
|
||||
if def.SessionID != "template-{{.template_session}}" {
|
||||
t.Fatalf("repository-owned prompt definition was mutated: %q", def.SessionID)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("direct value bypasses failing session template without changing messages", func(t *testing.T) {
|
||||
def := promptDef(domain.FormatText, domain.ValidationNone, 0)
|
||||
def.SessionID = "{{.missing_session}}"
|
||||
def.Templates = []domain.PromptMessageTemplate{
|
||||
{Role: "user", Content: "Hello {{.name}}"},
|
||||
}
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: def},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
nil,
|
||||
defaultArtifactReader(),
|
||||
prompt.NewGoRenderer(),
|
||||
&fakeLLM{forbid: true},
|
||||
nil,
|
||||
)
|
||||
|
||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
SessionID: "direct-session",
|
||||
Inputs: singleInputRef(),
|
||||
Vars: map[string]string{"name": "Rin"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare with direct session: %v", err)
|
||||
}
|
||||
if prepared.SessionID != "direct-session" {
|
||||
t.Fatalf("prepared session id = %q, want direct-session", prepared.SessionID)
|
||||
}
|
||||
if len(prepared.Messages) != 1 || prepared.Messages[0].Content != "Hello Rin" {
|
||||
t.Fatalf("message templates did not render normally: %+v", prepared.Messages)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("blank direct value retains prompt template behavior", func(t *testing.T) {
|
||||
def := promptDef(domain.FormatText, domain.ValidationNone, 0)
|
||||
def.SessionID = " template-{{.template_session}} "
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: def},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
nil,
|
||||
defaultArtifactReader(),
|
||||
prompt.NewGoRenderer(),
|
||||
&fakeLLM{forbid: true},
|
||||
nil,
|
||||
)
|
||||
|
||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
SessionID: " \t ",
|
||||
Inputs: singleInputRef(),
|
||||
Vars: map[string]string{"template_session": "rendered"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare with prompt session template: %v", err)
|
||||
}
|
||||
if prepared.SessionID != "template-rendered" {
|
||||
t.Fatalf("prepared session id = %q, want template-rendered", prepared.SessionID)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("overlong direct value fails before loading or generation", func(t *testing.T) {
|
||||
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "unexpected"}}
|
||||
runner := NewRunner(
|
||||
promptRepo,
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
nil,
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
llmClient,
|
||||
nil,
|
||||
)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
SessionID: strings.Repeat("界", domain.SessionIDMaxLength+1),
|
||||
Inputs: singleInputRef(),
|
||||
})
|
||||
if !errors.Is(err, ErrInvalidRequest) {
|
||||
t.Fatalf("expected ErrInvalidRequest, got %v", err)
|
||||
}
|
||||
if promptRepo.lastID != "" {
|
||||
t.Fatalf("invalid direct session loaded prompt %q", promptRepo.lastID)
|
||||
}
|
||||
if llmClient.calls != 0 {
|
||||
t.Fatalf("invalid direct session invoked generation %d times", llmClient.calls)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestRunnerPrepareUsesPromptDefaultProfileWhenNoExplicitProfileID(t *testing.T) {
|
||||
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
||||
promptRepo.def.DefaultProfile = "from-prompt"
|
||||
@@ -257,7 +408,7 @@ func TestRunnerPrepareSelectedProfileDoesNotExistFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunnerPreparePromptLoadFailure(t *testing.T) {
|
||||
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
|
||||
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
|
||||
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p"})
|
||||
if !errors.Is(err, ErrPromptLoad) {
|
||||
t.Fatalf("expected ErrPromptLoad, got %v", err)
|
||||
@@ -281,7 +432,7 @@ func TestRunnerPrepareRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) {
|
||||
ServiceTier: "priority",
|
||||
},
|
||||
}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||
|
||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -379,12 +530,12 @@ func TestRunnerPrepareRequestNumericOverridePresence(t *testing.T) {
|
||||
TopP: 0.8,
|
||||
TimeoutSeconds: 45,
|
||||
},
|
||||
}},
|
||||
}}, nil,
|
||||
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
&fakeLLM{forbid: true},
|
||||
nil,
|
||||
)
|
||||
nil)
|
||||
|
||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -426,12 +577,12 @@ func TestRunnerPrepareInvalidRequestNumericOverridesFail(t *testing.T) {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
&fakeLLM{forbid: true},
|
||||
nil,
|
||||
)
|
||||
nil)
|
||||
|
||||
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -458,7 +609,7 @@ func TestRunnerPrepareSelectedProfileBeatsBuiltInDefault(t *testing.T) {
|
||||
ServiceTier: "priority",
|
||||
},
|
||||
}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||
|
||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -495,12 +646,12 @@ func TestRunnerPrepareFileBackedPromptBodiesRenderCorrectly(t *testing.T) {
|
||||
llmClient := &fakeLLM{forbid: true}
|
||||
runner := NewRunner(
|
||||
promptdef.NewFilesystemRepository(promptDir),
|
||||
profile.NewFilesystemRepository(profileDir),
|
||||
profile.NewFilesystemRepository(profileDir), nil,
|
||||
|
||||
reader,
|
||||
prompt.NewGoRenderer(),
|
||||
llmClient,
|
||||
nil,
|
||||
)
|
||||
nil)
|
||||
|
||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||
PromptID: "valid-file-backed",
|
||||
@@ -526,12 +677,13 @@ func TestRunnerPrepareRequiredInputMissingFails(t *testing.T) {
|
||||
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: def},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||
|
||||
defaultArtifactReader(),
|
||||
prompt.NewGoRenderer(),
|
||||
&fakeLLM{forbid: true},
|
||||
nil,
|
||||
)
|
||||
nil)
|
||||
|
||||
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
@@ -552,12 +704,13 @@ func TestRunnerPrepareUnknownTemplateInputReferenceFails(t *testing.T) {
|
||||
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: def},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||
|
||||
defaultArtifactReader(),
|
||||
prompt.NewGoRenderer(),
|
||||
&fakeLLM{forbid: true},
|
||||
nil,
|
||||
)
|
||||
nil)
|
||||
|
||||
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
@@ -579,7 +732,7 @@ func TestRunnerPrepareAPIKeyEnvNameIncludedButNotResolvedValue(t *testing.T) {
|
||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName},
|
||||
}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||
|
||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||
if err != nil {
|
||||
@@ -607,12 +760,12 @@ func TestRunnerPrepareJSONSchemaBuildsStructuredOutputSpec(t *testing.T) {
|
||||
}
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: def},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
&fakeLLM{forbid: true},
|
||||
validator,
|
||||
)
|
||||
validator)
|
||||
|
||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -651,12 +804,12 @@ func TestRunnerPrepareJSONSchemaSchemaLoadFailureReturnsValidationError(t *testi
|
||||
validator := &fakeValidator{schemaErr: errors.New("schema unavailable")}
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: def},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
&fakeLLM{forbid: true},
|
||||
validator,
|
||||
)
|
||||
validator)
|
||||
|
||||
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -681,12 +834,12 @@ func TestRunnerRunJSONSchemaSchemaLoadFailureFailsBeforeLLM(t *testing.T) {
|
||||
validator := &fakeValidator{schemaErr: errors.New("schema unavailable")}
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: def},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
llmClient,
|
||||
validator,
|
||||
)
|
||||
validator)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -836,7 +989,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
|
||||
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}}
|
||||
|
||||
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil)
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
PromptVersion: "1",
|
||||
@@ -886,6 +1039,9 @@ func TestRunnerRunSuccessful(t *testing.T) {
|
||||
if llmClient.lastReq.Prompt.SessionID != "session-123" {
|
||||
t.Fatalf("expected session id to be sent to llm, got %q", llmClient.lastReq.Prompt.SessionID)
|
||||
}
|
||||
if res.SessionID != "session-123" {
|
||||
t.Fatalf("expected session id in run result, got %q", res.SessionID)
|
||||
}
|
||||
if res.Usage.TotalTokens != 7 {
|
||||
t.Fatalf("expected token usage to be retained, got %+v", res.Usage)
|
||||
}
|
||||
@@ -915,7 +1071,7 @@ func TestRunnerRunPassesExtraParamsToGenerateRequestTarget(t *testing.T) {
|
||||
},
|
||||
}}
|
||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -941,7 +1097,7 @@ func TestRunnerRunAndPrepareResolveSameProfileAndEffectiveSettings(t *testing.T)
|
||||
}}
|
||||
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap"}}
|
||||
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil)
|
||||
|
||||
req := domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -1071,7 +1227,7 @@ func TestRunnerRunExplicitRuntimeOverrideBeatsSelectedProfileValue(t *testing.T)
|
||||
},
|
||||
}}
|
||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -1115,7 +1271,7 @@ func TestRunnerRunSelectedProfileBeatsBuiltInDefault(t *testing.T) {
|
||||
},
|
||||
}}
|
||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -1142,7 +1298,7 @@ func TestRunnerRunBuiltInDefaultsUsedWhenProfileOmitsOptionalFields(t *testing.T
|
||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"},
|
||||
}}
|
||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -1172,7 +1328,7 @@ func TestRunnerRunAPIKeyEnvResolvesFromEnvironment(t *testing.T) {
|
||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_TEST_API_KEY"},
|
||||
}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||
if err != nil {
|
||||
@@ -1188,7 +1344,7 @@ func TestRunnerRunAPIKeyEnvMissingEnvironmentValueFailsClearly(t *testing.T) {
|
||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_MISSING_KEY"},
|
||||
}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||
if !errors.Is(err, ErrInvalidRequest) {
|
||||
@@ -1209,7 +1365,7 @@ func TestRunnerRunDirectAPIKeyBypassesMissingEnvAndReachesLLM(t *testing.T) {
|
||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_MISSING_KEY"},
|
||||
}}
|
||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -1233,7 +1389,7 @@ func TestRunnerPrepareAPIKeyRequiredFailsWithoutDirectKey(t *testing.T) {
|
||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true},
|
||||
}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||
|
||||
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -1252,7 +1408,7 @@ func TestRunnerRunAPIKeyRequiredSucceedsWithDirectKey(t *testing.T) {
|
||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true},
|
||||
}}
|
||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -1279,7 +1435,7 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideWorks(t *testing.T) {
|
||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"},
|
||||
}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -1304,7 +1460,7 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideBeatsProfile(t *testing.T) {
|
||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: profileEnv},
|
||||
}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -1328,7 +1484,7 @@ func TestRunnerRunAPIKeyValueNotPresentInMetadata(t *testing.T) {
|
||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName},
|
||||
}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||
if err != nil {
|
||||
@@ -1344,7 +1500,7 @@ func TestRunnerRunAPIKeyValueNotPresentInMetadata(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunnerRunPromptLoadFailure(t *testing.T) {
|
||||
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
|
||||
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"})
|
||||
if !errors.Is(err, ErrPromptLoad) {
|
||||
t.Fatalf("expected ErrPromptLoad, got %v", err)
|
||||
@@ -1357,12 +1513,12 @@ func TestRunnerRunPromptLoadFailure(t *testing.T) {
|
||||
func TestRunnerRunArtifactLoadFailure(t *testing.T) {
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||
|
||||
&fakeArtifactReader{errByURI: map[string]error{"a://bad": errors.New("read failed")}},
|
||||
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
|
||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
|
||||
nil,
|
||||
)
|
||||
nil)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -1377,12 +1533,13 @@ func TestRunnerRunArtifactLoadFailure(t *testing.T) {
|
||||
func TestRunnerRunPromptRenderFailure(t *testing.T) {
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||
|
||||
defaultArtifactReader(),
|
||||
&fakeRenderer{err: errors.New("render failed")},
|
||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
|
||||
nil,
|
||||
)
|
||||
nil)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
@@ -1396,12 +1553,13 @@ func TestRunnerRunPromptRenderFailure(t *testing.T) {
|
||||
func TestRunnerRunLLMFailure(t *testing.T) {
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
&fakeLLM{err: errors.New("llm failed")},
|
||||
nil,
|
||||
)
|
||||
nil)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
@@ -1415,12 +1573,13 @@ func TestRunnerRunLLMFailure(t *testing.T) {
|
||||
func TestRunnerRunCancellationPreservesGenerationCategory(t *testing.T) {
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "ignored"}},
|
||||
nil,
|
||||
)
|
||||
nil)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
@@ -1440,12 +1599,13 @@ func TestRunnerRunCancellationPreservesGenerationCategory(t *testing.T) {
|
||||
func TestRunnerRunLLMInvalidRequestMapsToUsecaseInvalidRequest(t *testing.T) {
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
&fakeLLM{err: llm.ErrInvalidRequest},
|
||||
nil,
|
||||
)
|
||||
nil)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
@@ -1463,12 +1623,13 @@ func TestRunnerRunValidationStillWorks(t *testing.T) {
|
||||
validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}}
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}},
|
||||
validator,
|
||||
)
|
||||
validator)
|
||||
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
@@ -1489,14 +1650,17 @@ func TestRunnerRunStructuredRepairRemainsBoundedAndUsesEffectiveModelSettings(t
|
||||
runner := NewRunnerWithRepairer(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", TimeoutSeconds: 55},
|
||||
"exec": {ID: "exec", BackendID: "custom", Model: "profile-model", TimeoutSeconds: 55},
|
||||
}}, fakeBackendResolver{backends: map[string]domain.Backend{
|
||||
"custom": {ID: "custom", Endpoint: "http://backend/v1"},
|
||||
}},
|
||||
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
llmClient,
|
||||
validate.NewStandardValidator("."),
|
||||
repairer,
|
||||
)
|
||||
repairer)
|
||||
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
@@ -1518,6 +1682,47 @@ func TestRunnerRunStructuredRepairRemainsBoundedAndUsesEffectiveModelSettings(t
|
||||
if repairer.reqs[0].Target.TimeoutSeconds != 22 {
|
||||
t.Fatalf("expected repair to use effective timeout, got %d", repairer.reqs[0].Target.TimeoutSeconds)
|
||||
}
|
||||
if llmClient.lastReq.Target.BackendID != "custom" ||
|
||||
repairer.reqs[0].Target.BackendID != "custom" ||
|
||||
res.SelectedBackendID != "custom" {
|
||||
t.Fatalf("expected backend identity in generation, repair, and result: generate=%q repair=%q result=%q",
|
||||
llmClient.lastReq.Target.BackendID, repairer.reqs[0].Target.BackendID, res.SelectedBackendID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunRepairCarriesEffectiveSessionID(t *testing.T) {
|
||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: `{"broken":`}}
|
||||
runner := NewRunnerWithRepairer(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||
"exec": {ID: "exec", Endpoint: "http://example.test/v1", Model: "model"},
|
||||
}},
|
||||
nil,
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
llmClient,
|
||||
validate.NewStandardValidator("."),
|
||||
NewDefaultOutputRepairer(llmClient),
|
||||
)
|
||||
|
||||
result, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
SessionID: " repair-session ",
|
||||
Inputs: singleInputRef(),
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
if llmClient.calls != 2 {
|
||||
t.Fatalf("expected initial generation and one repair, got %d calls", llmClient.calls)
|
||||
}
|
||||
if llmClient.lastReq.Prompt.SessionID != "repair-session" {
|
||||
t.Fatalf("expected repair generation to retain effective session, got %q", llmClient.lastReq.Prompt.SessionID)
|
||||
}
|
||||
if result.SessionID != "repair-session" {
|
||||
t.Fatalf("expected result to retain effective session, got %q", result.SessionID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunJSONSchemaRepairCarriesStructuredOutputSpec(t *testing.T) {
|
||||
@@ -1546,13 +1751,13 @@ func TestRunnerRunJSONSchemaRepairCarriesStructuredOutputSpec(t *testing.T) {
|
||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: `{"events":[1]}`}}
|
||||
runner := NewRunnerWithRepairer(
|
||||
&fakePromptRepo{def: def},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
llmClient,
|
||||
validator,
|
||||
repairer,
|
||||
)
|
||||
repairer)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -1629,13 +1834,12 @@ func TestResolveExecutionTargetProfileValuesPopulateAllSupportedFields(t *testin
|
||||
ServiceTier: "priority",
|
||||
ReasoningEffort: "low",
|
||||
APIKeyEnv: "PROFILE_KEY",
|
||||
APIKeyRequired: true,
|
||||
ExtraParams: map[string]any{
|
||||
"profile_option": "enabled",
|
||||
},
|
||||
}
|
||||
|
||||
target, presence, err := resolveExecutionTarget(profileValue, nil)
|
||||
target, presence, err := resolveExecutionTarget(nil, profileValue, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
@@ -1683,14 +1887,14 @@ func TestResolveExecutionTargetRuntimeOverridesBeatProfileForAllOverrideableFiel
|
||||
TopP: float64Ptr(0.5),
|
||||
TimeoutSeconds: intPtr(30),
|
||||
ServiceTier: "flex",
|
||||
ReasoningEffort: "high",
|
||||
ReasoningEffort: stringPtr("high"),
|
||||
APIKeyEnv: "RUNTIME_KEY",
|
||||
ExtraParams: map[string]any{
|
||||
"runtime_only": "yes",
|
||||
},
|
||||
}
|
||||
|
||||
target, presence, err := resolveExecutionTarget(profileValue, override)
|
||||
target, presence, err := resolveExecutionTarget(nil, profileValue, override)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
@@ -1704,7 +1908,7 @@ func TestResolveExecutionTargetRuntimeOverridesBeatProfileForAllOverrideableFiel
|
||||
target.TopP != *override.TopP ||
|
||||
target.TimeoutSeconds != *override.TimeoutSeconds ||
|
||||
target.ServiceTier != override.ServiceTier ||
|
||||
target.ReasoningEffort != override.ReasoningEffort ||
|
||||
target.ReasoningEffort != *override.ReasoningEffort ||
|
||||
target.APIKeyEnv != override.APIKeyEnv {
|
||||
t.Fatalf("expected runtime overrides to win for all fields, got %+v", target)
|
||||
}
|
||||
@@ -1713,6 +1917,46 @@ func TestResolveExecutionTargetRuntimeOverridesBeatProfileForAllOverrideableFiel
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveExecutionTargetReasoningOverrideStates(t *testing.T) {
|
||||
profileValue := &domain.ExecutionProfile{
|
||||
ReasoningEffort: "medium",
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
override *string
|
||||
want string
|
||||
}{
|
||||
{
|
||||
name: "nil inherits profile value",
|
||||
want: "medium",
|
||||
},
|
||||
{
|
||||
name: "nonblank replaces and trims profile value",
|
||||
override: stringPtr(" high "),
|
||||
want: "high",
|
||||
},
|
||||
{
|
||||
name: "blank clears profile value",
|
||||
override: stringPtr(" \t "),
|
||||
want: "",
|
||||
},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
target, _, err := resolveExecutionTarget(nil, profileValue, &domain.ExecutionTargetOverride{
|
||||
ReasoningEffort: tt.override,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("resolve execution target: %v", err)
|
||||
}
|
||||
if target.ReasoningEffort != tt.want {
|
||||
t.Fatalf("reasoning effort = %q, want %q", target.ReasoningEffort, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestMergeExecutionTargetEmptyStringOverridesDoNotErase(t *testing.T) {
|
||||
base := domain.ExecutionTarget{
|
||||
Endpoint: "http://base/v1",
|
||||
@@ -1813,6 +2057,118 @@ func defaultExecutionProfile() *domain.ExecutionProfile {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveExecutionTargetUsesBackendProfileAndRequestPrecedence(t *testing.T) {
|
||||
backendValue := &domain.Backend{
|
||||
ID: "custom",
|
||||
Endpoint: "http://backend/v1",
|
||||
APIKeyEnv: "BACKEND_KEY",
|
||||
ExtraParams: map[string]any{"backend": true},
|
||||
}
|
||||
profileValue := &domain.ExecutionProfile{
|
||||
ID: "exec",
|
||||
BackendID: "custom",
|
||||
Endpoint: "http://profile/v1",
|
||||
Model: "profile-model",
|
||||
APIKeyEnv: "PROFILE_KEY",
|
||||
ExtraParams: map[string]any{"profile": true},
|
||||
}
|
||||
override := &domain.ExecutionTargetOverride{
|
||||
Endpoint: "http://request/v1",
|
||||
APIKeyEnv: "REQUEST_KEY",
|
||||
ExtraParams: map[string]any{"request": true},
|
||||
}
|
||||
|
||||
target, _, err := resolveExecutionTarget(backendValue, profileValue, override)
|
||||
if err != nil {
|
||||
t.Fatalf("resolve target: %v", err)
|
||||
}
|
||||
if target.BackendID != "custom" {
|
||||
t.Fatalf("endpoint override changed backend identity: %+v", target)
|
||||
}
|
||||
if target.Endpoint != "http://request/v1" || target.APIKeyEnv != "REQUEST_KEY" {
|
||||
t.Fatalf("request values did not win: %+v", target)
|
||||
}
|
||||
if !reflect.DeepEqual(target.ExtraParams, map[string]any{"request": true}) {
|
||||
t.Fatalf("expected whole-map request replacement, got %#v", target.ExtraParams)
|
||||
}
|
||||
|
||||
target, _, err = resolveExecutionTarget(backendValue, &domain.ExecutionProfile{
|
||||
ID: "exec", BackendID: "custom", Model: "profile-model",
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("resolve backend defaults: %v", err)
|
||||
}
|
||||
if target.Endpoint != backendValue.Endpoint ||
|
||||
target.APIKeyEnv != backendValue.APIKeyEnv ||
|
||||
!reflect.DeepEqual(target.ExtraParams, backendValue.ExtraParams) {
|
||||
t.Fatalf("backend defaults were not inherited: %+v", target)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerPrepareBackendResolutionAndCredentialPrecedence(t *testing.T) {
|
||||
resolver := fakeBackendResolver{backends: map[string]domain.Backend{
|
||||
"custom": {ID: "custom", Endpoint: "http://backend/v1", APIKeyEnv: "BACKEND_KEY"},
|
||||
}}
|
||||
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
||||
|
||||
t.Run("unknown backend is a profile load failure", func(t *testing.T) {
|
||||
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||
"exec": {ID: "exec", BackendID: "unknown", Model: "model"},
|
||||
}}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||
if !errors.Is(err, ErrProfileLoad) {
|
||||
t.Fatalf("expected ErrProfileLoad, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("nil resolver is a profile load failure", func(t *testing.T) {
|
||||
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||
"exec": {ID: "exec", BackendID: "custom", Model: "model"},
|
||||
}}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||
if !errors.Is(err, ErrProfileLoad) {
|
||||
t.Fatalf("expected ErrProfileLoad, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("request environment wins", func(t *testing.T) {
|
||||
t.Setenv("REQUEST_KEY", "request-secret")
|
||||
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||
"exec": {ID: "exec", BackendID: "custom", Model: "model", APIKeyEnv: "PROFILE_KEY"},
|
||||
}}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||
PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(),
|
||||
Execution: &domain.ExecutionTargetOverride{APIKeyEnv: "REQUEST_KEY"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare: %v", err)
|
||||
}
|
||||
if prepared.EffectiveModelParams.APIKeyEnv != "REQUEST_KEY" {
|
||||
t.Fatalf("unexpected credential source: %+v", prepared.EffectiveModelParams)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("required direct key clears inherited environment", func(t *testing.T) {
|
||||
t.Setenv("BACKEND_KEY", "backend-secret")
|
||||
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||
"exec": {ID: "exec", BackendID: "custom", Model: "model", APIKeyRequired: true},
|
||||
}}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||
if !errors.Is(err, ErrAPIKeyRequired) {
|
||||
t.Fatalf("expected ErrAPIKeyRequired, got %v", err)
|
||||
}
|
||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||
PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(), APIKey: "direct-secret",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare with direct key: %v", err)
|
||||
}
|
||||
if prepared.EffectiveModelParams.APIKeyEnv != "" {
|
||||
t.Fatalf("expected inherited environment name to be cleared, got %q", prepared.EffectiveModelParams.APIKeyEnv)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func defaultArtifactReader() *fakeArtifactReader {
|
||||
return &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{
|
||||
"a://ok": {Body: []byte("x"), Hash: hashString("x")},
|
||||
@@ -1831,6 +2187,10 @@ func float64Ptr(v float64) *float64 {
|
||||
return &v
|
||||
}
|
||||
|
||||
func stringPtr(v string) *string {
|
||||
return &v
|
||||
}
|
||||
|
||||
func intPtr(v int) *int {
|
||||
return &v
|
||||
}
|
||||
@@ -1838,10 +2198,11 @@ func intPtr(v int) *int {
|
||||
func newMinimalRunner(promptRepo *fakePromptRepo, execRepo *fakeExecutionProfileRepo) *Runner {
|
||||
return NewRunner(
|
||||
promptRepo,
|
||||
execRepo,
|
||||
execRepo, nil,
|
||||
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
|
||||
nil,
|
||||
)
|
||||
nil)
|
||||
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/fs"
|
||||
"net/url"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
@@ -16,6 +17,8 @@ import (
|
||||
"github.com/santhosh-tekuri/jsonschema/v6"
|
||||
)
|
||||
|
||||
const jsonSchemaDraft2020 = "https://json-schema.org/draft/2020-12/schema"
|
||||
|
||||
// StandardValidator provides basic, JSON, and JSON Schema output validation.
|
||||
type StandardValidator struct {
|
||||
schemaBaseDir string
|
||||
@@ -121,7 +124,11 @@ func (v *StandardValidator) validateJSONSchema(instance any, schemaPath string)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
compiler := jsonschema.NewCompiler()
|
||||
schemaRoot, err := v.schemaRoot()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
compiler := newSchemaCompiler(standardSchemaLoader{root: schemaRoot})
|
||||
schema, err := compiler.Compile(resolvedSchemaPath)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", resolvedSchemaPath, err)
|
||||
@@ -140,7 +147,10 @@ func (v *FSValidator) validateJSONSchema(instance any, schemaPath string) ([]str
|
||||
}
|
||||
|
||||
resourceURL := fsSchemaResourceURL(schemaName)
|
||||
compiler := jsonschema.NewCompiler()
|
||||
if err := validateSchemaDialect(schemaDoc); err != nil {
|
||||
return nil, fmt.Errorf("failed to compile JSON schema %q: %w", schemaName, err)
|
||||
}
|
||||
compiler := newSchemaCompiler(fsSchemaLoader{fsys: v.fsys, root: filecatalog.CleanFSRoot(v.root)})
|
||||
if err := compiler.AddResource(resourceURL, schemaDoc); err != nil {
|
||||
return nil, fmt.Errorf("failed to register JSON schema %q: %w", schemaName, err)
|
||||
}
|
||||
@@ -184,6 +194,9 @@ func (v *StandardValidator) LoadSchemaDocument(ctx context.Context, schemaPath s
|
||||
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||
return nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err)
|
||||
}
|
||||
if err := validateSchemaDialect(doc); err != nil {
|
||||
return nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err)
|
||||
}
|
||||
return doc, nil
|
||||
}
|
||||
|
||||
@@ -206,12 +219,14 @@ func (v *StandardValidator) resolveSchemaPath(schemaPath string) (string, error)
|
||||
return "", errors.New("schema path is required for json_schema validation")
|
||||
}
|
||||
|
||||
resolved := schemaPath
|
||||
if !filepath.IsAbs(schemaPath) {
|
||||
resolved = filepath.Join(v.schemaBaseDir, schemaPath)
|
||||
root, err := v.schemaRoot()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
resolved, err := containedFilesystemPath(root, schemaPath)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
resolved = filepath.Clean(resolved)
|
||||
if _, err := os.Stat(resolved); err != nil {
|
||||
return "", fmt.Errorf("failed to access schema file %q: %w", resolved, err)
|
||||
}
|
||||
@@ -219,6 +234,22 @@ func (v *StandardValidator) resolveSchemaPath(schemaPath string) (string, error)
|
||||
return resolved, nil
|
||||
}
|
||||
|
||||
func (v *StandardValidator) schemaRoot() (string, error) {
|
||||
root := v.schemaBaseDir
|
||||
if strings.TrimSpace(root) == "" {
|
||||
root = "."
|
||||
}
|
||||
absolute, err := filepath.Abs(root)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to resolve schema source %q: %w", root, err)
|
||||
}
|
||||
resolved, err := filepath.EvalSymlinks(absolute)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to access schema source %q: %w", root, err)
|
||||
}
|
||||
return resolved, nil
|
||||
}
|
||||
|
||||
func (v *FSValidator) loadSchemaDocument(schemaPath string) (string, any, error) {
|
||||
resolved, err := v.resolveSchemaPath(schemaPath)
|
||||
if err != nil {
|
||||
@@ -234,6 +265,9 @@ func (v *FSValidator) loadSchemaDocument(schemaPath string) (string, any, error)
|
||||
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||
return "", nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err)
|
||||
}
|
||||
if err := validateSchemaDialect(doc); err != nil {
|
||||
return "", nil, fmt.Errorf("failed to decode JSON schema %q: %w", resolved, err)
|
||||
}
|
||||
return resolved, doc, nil
|
||||
}
|
||||
|
||||
@@ -290,3 +324,131 @@ func cleanSchemaFSPath(schemaPath string) (string, error) {
|
||||
func fsSchemaResourceURL(schemaName string) string {
|
||||
return "promptkit-schema:///" + strings.TrimPrefix(path.Clean(schemaName), "/")
|
||||
}
|
||||
|
||||
func newSchemaCompiler(loader jsonschema.URLLoader) *jsonschema.Compiler {
|
||||
compiler := jsonschema.NewCompiler()
|
||||
compiler.DefaultDraft(jsonschema.Draft2020)
|
||||
compiler.UseLoader(loader)
|
||||
return compiler
|
||||
}
|
||||
|
||||
func validateSchemaDialect(doc any) error {
|
||||
object, ok := doc.(map[string]any)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
value, ok := object["$schema"]
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
dialect, ok := value.(string)
|
||||
if !ok {
|
||||
return errors.New("$schema must be a string")
|
||||
}
|
||||
if dialect != jsonSchemaDraft2020 && dialect != jsonSchemaDraft2020+"#" {
|
||||
return fmt.Errorf("unsupported JSON Schema dialect %q; expected %q", dialect, jsonSchemaDraft2020)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type standardSchemaLoader struct {
|
||||
root string
|
||||
}
|
||||
|
||||
func (l standardSchemaLoader) Load(resourceURL string) (any, error) {
|
||||
fileName, err := (jsonschema.FileLoader{}).ToFile(resourceURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("schema reference %q is not a contained file reference: %w", resourceURL, err)
|
||||
}
|
||||
resolved, err := containedFilesystemPath(l.root, fileName)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return loadJSONSchemaFile(resolved)
|
||||
}
|
||||
|
||||
func containedFilesystemPath(root, name string) (string, error) {
|
||||
candidate := name
|
||||
if !filepath.IsAbs(candidate) {
|
||||
candidate = filepath.Join(root, candidate)
|
||||
}
|
||||
candidate, err := filepath.Abs(candidate)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to resolve schema path %q: %w", name, err)
|
||||
}
|
||||
candidate, err = filepath.EvalSymlinks(candidate)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to access schema file %q: %w", candidate, err)
|
||||
}
|
||||
relative, err := filepath.Rel(root, candidate)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("failed to compare schema path %q with source root: %w", candidate, err)
|
||||
}
|
||||
if relative == ".." || strings.HasPrefix(relative, ".."+string(filepath.Separator)) {
|
||||
return "", fmt.Errorf("schema path %q escapes source root", name)
|
||||
}
|
||||
return candidate, nil
|
||||
}
|
||||
|
||||
func loadJSONSchemaFile(name string) (any, error) {
|
||||
raw, err := os.ReadFile(name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var doc any
|
||||
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := validateSchemaDialect(doc); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return doc, nil
|
||||
}
|
||||
|
||||
type fsSchemaLoader struct {
|
||||
fsys fs.FS
|
||||
root string
|
||||
}
|
||||
|
||||
func (l fsSchemaLoader) Load(resourceURL string) (any, error) {
|
||||
parsed, err := url.Parse(resourceURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid schema reference %q: %w", resourceURL, err)
|
||||
}
|
||||
if parsed.Scheme != "promptkit-schema" || parsed.Host != "" {
|
||||
return nil, fmt.Errorf("schema reference %q is not allowed", resourceURL)
|
||||
}
|
||||
name, err := url.PathUnescape(strings.TrimPrefix(parsed.Path, "/"))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("invalid schema reference %q: %w", resourceURL, err)
|
||||
}
|
||||
name = path.Clean(name)
|
||||
if l.root == "." {
|
||||
if strings.HasPrefix(name, "../") || name == ".." {
|
||||
return nil, fmt.Errorf("schema reference %q escapes source root", resourceURL)
|
||||
}
|
||||
} else if name != l.root && !strings.HasPrefix(name, l.root+"/") {
|
||||
return nil, fmt.Errorf("schema reference %q escapes source root", resourceURL)
|
||||
}
|
||||
|
||||
rootInfo, err := fs.Stat(l.fsys, l.root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !rootInfo.IsDir() && name != l.root {
|
||||
return nil, fmt.Errorf("schema reference %q is outside the configured schema file", resourceURL)
|
||||
}
|
||||
|
||||
raw, err := fs.ReadFile(l.fsys, name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var doc any
|
||||
if err := json.Unmarshal(raw, &doc); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := validateSchemaDialect(doc); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return doc, nil
|
||||
}
|
||||
|
||||
@@ -2,8 +2,10 @@ package validate
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
@@ -411,3 +413,138 @@ func TestFSValidatorLoadSchemaDocument(t *testing.T) {
|
||||
t.Fatalf("unexpected schema document: %#v", doc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStandardValidatorJSONSchemaReferenceBoundaries(t *testing.T) {
|
||||
root := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(root, "child.json"), []byte(`{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"type": "string"
|
||||
}`), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
tests := []struct {
|
||||
name string
|
||||
reference string
|
||||
wantError string
|
||||
writeOuter bool
|
||||
}{
|
||||
{name: "contained relative reference", reference: "child.json"},
|
||||
{name: "remote reference", reference: "https://example.test/schema.json", wantError: "not a contained file reference"},
|
||||
{name: "escaping reference", reference: "../outside.json", wantError: "escapes source root", writeOuter: true},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if tc.writeOuter {
|
||||
if err := os.WriteFile(filepath.Join(filepath.Dir(root), "outside.json"), []byte(`{"type":"string"}`), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
schema := `{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$ref": ` + strconv.Quote(tc.reference) + `
|
||||
}`
|
||||
if err := os.WriteFile(filepath.Join(root, "root.json"), []byte(schema), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
v := NewStandardValidator(root)
|
||||
result, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`"value"`)}, domain.OutputContract{
|
||||
ValidationMode: domain.ValidationJSONSchema,
|
||||
SchemaPath: "root.json",
|
||||
})
|
||||
if tc.wantError == "" {
|
||||
if err != nil || !result.IsValid {
|
||||
t.Fatalf("expected contained reference to validate, got result=%#v error=%v", result, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), tc.wantError) {
|
||||
t.Fatalf("expected error containing %q, got %v", tc.wantError, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestFSValidatorJSONSchemaReferenceBoundaries(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
reference string
|
||||
wantError string
|
||||
}{
|
||||
{name: "same document fragment", reference: "#/$defs/value"},
|
||||
{name: "contained relative reference", reference: "child.json"},
|
||||
{name: "remote reference", reference: "https://example.test/schema.json", wantError: "is not allowed"},
|
||||
{name: "escaping reference", reference: "../outside.json", wantError: "escapes source root"},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
rootSchema := `{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$defs": {"value": {"type": "string"}},
|
||||
"$ref": ` + strconv.Quote(tc.reference) + `
|
||||
}`
|
||||
v := NewFSValidator(fstest.MapFS{
|
||||
"schemas/root.json": &fstest.MapFile{Data: []byte(rootSchema)},
|
||||
"schemas/child.json": &fstest.MapFile{Data: []byte(`{"type":"string"}`)},
|
||||
"outside.json": &fstest.MapFile{Data: []byte(`{"type":"string"}`)},
|
||||
}, "schemas")
|
||||
result, err := v.Validate(context.Background(), &domain.Artifact{Body: []byte(`"value"`)}, domain.OutputContract{
|
||||
ValidationMode: domain.ValidationJSONSchema,
|
||||
SchemaPath: "root.json",
|
||||
})
|
||||
if tc.wantError == "" {
|
||||
if err != nil || !result.IsValid {
|
||||
t.Fatalf("expected supported reference to validate, got result=%#v error=%v", result, err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err == nil || !strings.Contains(err.Error(), tc.wantError) {
|
||||
t.Fatalf("expected error containing %q, got %v", tc.wantError, err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestJSONSchemaDialectIsDraft2020(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
dialect string
|
||||
wantError bool
|
||||
}{
|
||||
{name: "omitted uses supported default"},
|
||||
{name: "draft 2020-12", dialect: "https://json-schema.org/draft/2020-12/schema"},
|
||||
{name: "draft 7 rejected", dialect: "http://json-schema.org/draft-07/schema#", wantError: true},
|
||||
}
|
||||
|
||||
for _, tc := range tests {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
schema := map[string]any{"type": "object"}
|
||||
if tc.dialect != "" {
|
||||
schema["$schema"] = tc.dialect
|
||||
}
|
||||
data, err := json.Marshal(schema)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
v := NewFSValidator(fstest.MapFS{
|
||||
"schema.json": &fstest.MapFile{Data: data},
|
||||
}, ".")
|
||||
_, err = v.Validate(context.Background(), &domain.Artifact{Body: []byte(`{}`)}, domain.OutputContract{
|
||||
ValidationMode: domain.ValidationJSONSchema,
|
||||
SchemaPath: "schema.json",
|
||||
})
|
||||
if tc.wantError {
|
||||
if err == nil || !strings.Contains(err.Error(), "unsupported JSON Schema dialect") {
|
||||
t.Fatalf("expected unsupported-dialect error, got %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("expected supported dialect, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
162
json.go
Normal file
162
json.go
Normal file
@@ -0,0 +1,162 @@
|
||||
package promptkit
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"time"
|
||||
)
|
||||
|
||||
// MarshalJSON implements json.Marshaler for PreparedRun. It uses RFC 3339
|
||||
// timestamps, integer duration_ms, and omits zero timing values.
|
||||
func (r PreparedRun) MarshalJSON() ([]byte, error) {
|
||||
var startTime, endTime *time.Time
|
||||
if !r.StartTime.IsZero() {
|
||||
startTime = &r.StartTime
|
||||
}
|
||||
if !r.EndTime.IsZero() {
|
||||
endTime = &r.EndTime
|
||||
}
|
||||
|
||||
var durationMS *int64
|
||||
if r.DurationMS != 0 {
|
||||
durationMS = &r.DurationMS
|
||||
}
|
||||
|
||||
return json.Marshal(struct {
|
||||
PromptID string `json:"prompt_id"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
PromptHash string `json:"prompt_hash,omitempty"`
|
||||
SelectedProfileID string `json:"selected_profile_id"`
|
||||
SelectedBackendID string `json:"selected_backend_id,omitempty"`
|
||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||
OutputContract OutputContract `json:"output_contract"`
|
||||
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||
Messages []RenderedMessage `json:"messages"`
|
||||
StartTime *time.Time `json:"start_time,omitempty"`
|
||||
EndTime *time.Time `json:"end_time,omitempty"`
|
||||
DurationMS *int64 `json:"duration_ms,omitempty"`
|
||||
}{
|
||||
PromptID: r.PromptID,
|
||||
PromptVersion: r.PromptVersion,
|
||||
PromptHash: r.PromptHash,
|
||||
SelectedProfileID: r.SelectedProfileID,
|
||||
SelectedBackendID: r.SelectedBackendID,
|
||||
EffectiveModelParams: r.EffectiveModelParams,
|
||||
OutputContract: r.OutputContract,
|
||||
StructuredOutput: r.StructuredOutput,
|
||||
InputHashes: r.InputHashes,
|
||||
SessionID: r.SessionID,
|
||||
RenderedPromptHash: r.RenderedPromptHash,
|
||||
Messages: r.Messages,
|
||||
StartTime: startTime,
|
||||
EndTime: endTime,
|
||||
DurationMS: durationMS,
|
||||
})
|
||||
}
|
||||
|
||||
// MarshalJSON implements json.Marshaler for RunResult. It encodes Duration as
|
||||
// integer milliseconds in duration_ms and omits zero timing values.
|
||||
func (r RunResult) MarshalJSON() ([]byte, error) {
|
||||
var startTime, endTime *time.Time
|
||||
if !r.StartTime.IsZero() {
|
||||
startTime = &r.StartTime
|
||||
}
|
||||
if !r.EndTime.IsZero() {
|
||||
endTime = &r.EndTime
|
||||
}
|
||||
|
||||
var durationMS *int64
|
||||
if r.Duration != 0 {
|
||||
value := r.Duration.Milliseconds()
|
||||
durationMS = &value
|
||||
}
|
||||
|
||||
return json.Marshal(runResultJSON{
|
||||
RunID: r.RunID,
|
||||
Artifact: r.Artifact,
|
||||
RawOutput: r.RawOutput,
|
||||
Validation: r.Validation,
|
||||
PromptID: r.PromptID,
|
||||
PromptVersion: r.PromptVersion,
|
||||
PromptHash: r.PromptHash,
|
||||
SessionID: r.SessionID,
|
||||
RenderedPromptHash: r.RenderedPromptHash,
|
||||
SelectedProfileID: r.SelectedProfileID,
|
||||
SelectedBackendID: r.SelectedBackendID,
|
||||
ModelName: r.ModelName,
|
||||
Endpoint: r.Endpoint,
|
||||
EffectiveModelParams: r.EffectiveModelParams,
|
||||
InputHashes: r.InputHashes,
|
||||
Usage: r.Usage,
|
||||
StartTime: startTime,
|
||||
EndTime: endTime,
|
||||
DurationMS: durationMS,
|
||||
})
|
||||
}
|
||||
|
||||
// UnmarshalJSON implements json.Unmarshaler for RunResult. It decodes
|
||||
// duration_ms into Duration with millisecond precision.
|
||||
func (r *RunResult) UnmarshalJSON(data []byte) error {
|
||||
var wire runResultJSON
|
||||
if err := json.Unmarshal(data, &wire); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
*r = RunResult{
|
||||
RunID: wire.RunID,
|
||||
Artifact: wire.Artifact,
|
||||
RawOutput: wire.RawOutput,
|
||||
Validation: wire.Validation,
|
||||
PromptID: wire.PromptID,
|
||||
PromptVersion: wire.PromptVersion,
|
||||
PromptHash: wire.PromptHash,
|
||||
SessionID: wire.SessionID,
|
||||
RenderedPromptHash: wire.RenderedPromptHash,
|
||||
SelectedProfileID: wire.SelectedProfileID,
|
||||
SelectedBackendID: wire.SelectedBackendID,
|
||||
ModelName: wire.ModelName,
|
||||
Endpoint: wire.Endpoint,
|
||||
EffectiveModelParams: wire.EffectiveModelParams,
|
||||
InputHashes: wire.InputHashes,
|
||||
Usage: wire.Usage,
|
||||
Duration: time.Duration(valueOrZero(wire.DurationMS)) * time.Millisecond,
|
||||
}
|
||||
if wire.StartTime != nil {
|
||||
r.StartTime = *wire.StartTime
|
||||
}
|
||||
if wire.EndTime != nil {
|
||||
r.EndTime = *wire.EndTime
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type runResultJSON struct {
|
||||
RunID string `json:"run_id"`
|
||||
Artifact Artifact `json:"artifact"`
|
||||
RawOutput string `json:"raw_output"`
|
||||
Validation ValidationResult `json:"validation"`
|
||||
PromptID string `json:"prompt_id"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
PromptHash string `json:"prompt_hash,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||
SelectedProfileID string `json:"selected_profile_id"`
|
||||
SelectedBackendID string `json:"selected_backend_id,omitempty"`
|
||||
ModelName string `json:"model_name"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||
Usage TokenUsage `json:"usage"`
|
||||
StartTime *time.Time `json:"start_time,omitempty"`
|
||||
EndTime *time.Time `json:"end_time,omitempty"`
|
||||
DurationMS *int64 `json:"duration_ms,omitempty"`
|
||||
}
|
||||
|
||||
func valueOrZero(value *int64) int64 {
|
||||
if value == nil {
|
||||
return 0
|
||||
}
|
||||
return *value
|
||||
}
|
||||
16
profiles.go
16
profiles.go
@@ -7,6 +7,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
)
|
||||
|
||||
@@ -15,10 +16,16 @@ import (
|
||||
//
|
||||
// It does not register global state, maintain a model catalog, or resolve
|
||||
// credentials. If APIKeyRequired is true, callers satisfy it with
|
||||
// RunRequest.APIKey. Raw API keys do not belong in profiles.
|
||||
// RunRequest.APIKey or an explicit request ExecutionTargetOverride.APIKeyEnv.
|
||||
// Raw API keys do not belong in profiles.
|
||||
//
|
||||
// The function copies the ExtraParams map itself but does not recursively copy
|
||||
// nested values. Validation and a deep copy occur when NewEngine applies a
|
||||
// WithProfiles option containing the returned Profile.
|
||||
func OpenAICompatibleProfile(cfg OpenAICompatibleProfileConfig) Profile {
|
||||
return Profile{
|
||||
ID: cfg.ID,
|
||||
BackendID: cfg.BackendID,
|
||||
Endpoint: cfg.Endpoint,
|
||||
Model: cfg.Model,
|
||||
Temperature: cfg.Temperature,
|
||||
@@ -75,12 +82,13 @@ func (r *memoryProfileRepository) GetProfile(_ context.Context, id string) (*dom
|
||||
}
|
||||
|
||||
func toDomainProfile(publicProfile Profile) (domain.ExecutionProfile, error) {
|
||||
extraParams, err := copyPublicJSONMap(publicProfile.ExtraParams)
|
||||
extraParams, err := jsonvalue.CopyMap(publicProfile.ExtraParams)
|
||||
if err != nil {
|
||||
return domain.ExecutionProfile{}, err
|
||||
}
|
||||
prof := domain.ExecutionProfile{
|
||||
ID: strings.TrimSpace(publicProfile.ID),
|
||||
BackendID: strings.TrimSpace(publicProfile.BackendID),
|
||||
Endpoint: publicProfile.Endpoint,
|
||||
Model: publicProfile.Model,
|
||||
Temperature: publicProfile.Temperature,
|
||||
@@ -102,8 +110,8 @@ func validatePublicProfile(prof domain.ExecutionProfile) error {
|
||||
if strings.TrimSpace(prof.ID) == "" {
|
||||
return errors.New("id is required")
|
||||
}
|
||||
if strings.TrimSpace(prof.Endpoint) == "" {
|
||||
return errors.New("endpoint is required")
|
||||
if strings.TrimSpace(prof.BackendID) == "" && strings.TrimSpace(prof.Endpoint) == "" {
|
||||
return errors.New("backend or endpoint is required")
|
||||
}
|
||||
if strings.TrimSpace(prof.Model) == "" {
|
||||
return errors.New("model is required")
|
||||
|
||||
746
public_contract_test.go
Normal file
746
public_contract_test.go
Normal file
@@ -0,0 +1,746 @@
|
||||
package promptkit_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"testing/fstest"
|
||||
"time"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit"
|
||||
)
|
||||
|
||||
func TestPreparedRunJSONOmitsZeroTimingValues(t *testing.T) {
|
||||
payload, err := json.Marshal(promptkit.PreparedRun{})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal prepared run: %v", err)
|
||||
}
|
||||
for _, field := range []string{"start_time", "end_time", "duration_ms"} {
|
||||
if strings.Contains(string(payload), `"`+field+`"`) {
|
||||
t.Fatalf("expected zero %s to be omitted, got %s", field, payload)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackendIdentityJSONNamesAndOmission(t *testing.T) {
|
||||
t.Run("execution target round trip", func(t *testing.T) {
|
||||
value := promptkit.ExecutionTarget{BackendID: promptkit.BackendOpenRouter}
|
||||
payload, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal execution target: %v", err)
|
||||
}
|
||||
var decoded promptkit.ExecutionTarget
|
||||
if err := json.Unmarshal(payload, &decoded); err != nil {
|
||||
t.Fatalf("unmarshal execution target: %v", err)
|
||||
}
|
||||
if decoded.BackendID != value.BackendID {
|
||||
t.Fatalf("backend identity did not round trip: got %q want %q", decoded.BackendID, value.BackendID)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("prepared run round trip", func(t *testing.T) {
|
||||
value := promptkit.PreparedRun{SelectedBackendID: promptkit.BackendOpenRouter}
|
||||
payload, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal prepared run: %v", err)
|
||||
}
|
||||
var decoded promptkit.PreparedRun
|
||||
if err := json.Unmarshal(payload, &decoded); err != nil {
|
||||
t.Fatalf("unmarshal prepared run: %v", err)
|
||||
}
|
||||
if decoded.SelectedBackendID != value.SelectedBackendID {
|
||||
t.Fatalf("backend identity did not round trip: got %q want %q", decoded.SelectedBackendID, value.SelectedBackendID)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("run result round trip", func(t *testing.T) {
|
||||
value := promptkit.RunResult{SelectedBackendID: promptkit.BackendOpenRouter}
|
||||
payload, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal run result: %v", err)
|
||||
}
|
||||
var decoded promptkit.RunResult
|
||||
if err := json.Unmarshal(payload, &decoded); err != nil {
|
||||
t.Fatalf("unmarshal run result: %v", err)
|
||||
}
|
||||
if decoded.SelectedBackendID != value.SelectedBackendID {
|
||||
t.Fatalf("backend identity did not round trip: got %q want %q", decoded.SelectedBackendID, value.SelectedBackendID)
|
||||
}
|
||||
})
|
||||
|
||||
payload, err := json.Marshal(promptkit.ExecutionTarget{})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal empty execution target: %v", err)
|
||||
}
|
||||
if strings.Contains(string(payload), `"backend_id"`) {
|
||||
t.Fatalf("empty backend identity was not omitted: %s", payload)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEndpointOnlyProfileOmitsBackendIdentityFromStableJSON(t *testing.T) {
|
||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "profile", Endpoint: "http://example.test/v1", Model: "model",
|
||||
}),
|
||||
promptkit.WithLLMClient(client),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare endpoint-only profile: %v", err)
|
||||
}
|
||||
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||
if err != nil {
|
||||
t.Fatalf("run endpoint-only profile: %v", err)
|
||||
}
|
||||
if prepared.SelectedBackendID != "" ||
|
||||
prepared.EffectiveModelParams.BackendID != "" ||
|
||||
result.SelectedBackendID != "" ||
|
||||
result.EffectiveModelParams.BackendID != "" {
|
||||
t.Fatalf("endpoint-only profile acquired backend identity: prepared=%+v result=%+v", prepared, result)
|
||||
}
|
||||
for _, value := range []any{prepared, result} {
|
||||
payload, err := json.Marshal(value)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal endpoint-only value: %v", err)
|
||||
}
|
||||
if strings.Contains(string(payload), `"backend_id"`) || strings.Contains(string(payload), `"selected_backend_id"`) {
|
||||
t.Fatalf("endpoint-only backend identity was not omitted: %s", payload)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnknownProfileBackendHasProfileLoadIdentity(t *testing.T) {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
|
||||
promptkit.WithProfiles(promptkit.Profile{ID: "profile", BackendID: "unknown", Model: "model"}),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
|
||||
_, err = engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||
if !errors.Is(err, promptkit.ErrProfileLoad) {
|
||||
t.Fatalf("expected ErrProfileLoad, got %v", err)
|
||||
}
|
||||
if errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||
t.Fatalf("unknown backend should not have invalid-request identity: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCustomBackendFlowsThroughProfilesOverridesAndInjectedClient(t *testing.T) {
|
||||
t.Setenv("CUSTOM_LLM_KEY", "test-key")
|
||||
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractPromptFS("prompt", "backend-profile", "message"), "."),
|
||||
promptkit.WithBackend(promptkit.Backend{
|
||||
ID: " custom ",
|
||||
Endpoint: " http://backend.example/v1 ",
|
||||
APIKeyEnv: " CUSTOM_LLM_KEY ",
|
||||
ExtraParams: map[string]any{
|
||||
"provider": "custom",
|
||||
},
|
||||
}),
|
||||
promptkit.WithProfiles(
|
||||
promptkit.Profile{ID: "backend-profile", BackendID: "custom", Model: "backend-model"},
|
||||
promptkit.Profile{ID: "profile-endpoint", BackendID: "custom", Endpoint: "http://profile.example/v1", Model: "profile-model"},
|
||||
promptkit.Profile{ID: "blank-profile-endpoint", BackendID: "custom", Endpoint: " \t ", Model: "profile-model"},
|
||||
),
|
||||
promptkit.WithLLMClient(client),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
|
||||
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||
if err != nil {
|
||||
t.Fatalf("run with custom backend: %v", err)
|
||||
}
|
||||
if len(client.requests) != 1 {
|
||||
t.Fatalf("expected one injected-client request, got %d", len(client.requests))
|
||||
}
|
||||
target := client.requests[0].Target
|
||||
if target.BackendID != "custom" ||
|
||||
target.Endpoint != "http://backend.example/v1" ||
|
||||
target.APIKeyEnv != "CUSTOM_LLM_KEY" ||
|
||||
target.Model != "backend-model" ||
|
||||
target.ExtraParams["provider"] != "custom" ||
|
||||
result.SelectedBackendID != "custom" {
|
||||
t.Fatalf("unexpected custom backend settings: target=%+v result_backend=%q", target, result.SelectedBackendID)
|
||||
}
|
||||
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "prompt", ProfileID: "profile-endpoint",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare profile endpoint override: %v", err)
|
||||
}
|
||||
if prepared.SelectedBackendID != "custom" || prepared.EffectiveModelParams.Endpoint != "http://profile.example/v1" {
|
||||
t.Fatalf("profile endpoint override changed backend identity: %+v", prepared)
|
||||
}
|
||||
|
||||
prepared, err = engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "prompt", ProfileID: "blank-profile-endpoint",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare blank profile endpoint: %v", err)
|
||||
}
|
||||
if prepared.SelectedBackendID != "custom" || prepared.EffectiveModelParams.Endpoint != "http://backend.example/v1" {
|
||||
t.Fatalf("blank profile endpoint did not inherit backend endpoint: %+v", prepared)
|
||||
}
|
||||
|
||||
prepared, err = engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "prompt",
|
||||
Execution: &promptkit.ExecutionTargetOverride{
|
||||
Endpoint: "http://request.example/v1",
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare request endpoint override: %v", err)
|
||||
}
|
||||
if prepared.SelectedBackendID != "custom" || prepared.EffectiveModelParams.Endpoint != "http://request.example/v1" {
|
||||
t.Fatalf("request endpoint override changed backend identity: %+v", prepared)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCustomBackendSupportsFileProfileAndBothSelectionPaths(t *testing.T) {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractPromptFS("prompt", "file-profile", "message"), "."),
|
||||
promptkit.WithProfileFS(fstest.MapFS{
|
||||
"profile.yaml": &fstest.MapFile{Data: []byte(`id: file-profile
|
||||
backend: file-backend
|
||||
endpoint: " "
|
||||
model: file-model
|
||||
`)},
|
||||
}, "."),
|
||||
promptkit.WithBackend(promptkit.Backend{
|
||||
ID: "file-backend",
|
||||
Endpoint: "http://file-backend.example/v1",
|
||||
}),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
|
||||
for _, request := range []promptkit.RunRequest{
|
||||
{PromptID: "prompt"},
|
||||
{PromptID: "prompt", ProfileID: "file-profile"},
|
||||
} {
|
||||
prepared, err := engine.Prepare(context.Background(), request)
|
||||
if err != nil {
|
||||
t.Fatalf("prepare file profile: %v", err)
|
||||
}
|
||||
if prepared.SelectedBackendID != "file-backend" ||
|
||||
prepared.EffectiveModelParams.Endpoint != "http://file-backend.example/v1" {
|
||||
t.Fatalf("unexpected file-profile backend resolution: %+v", prepared)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackendOptionsAccumulateAndRegistrationsAreEngineLocal(t *testing.T) {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractPromptFS("prompt", "first-profile", "message"), "."),
|
||||
promptkit.WithBackend(promptkit.Backend{ID: "first", Endpoint: "http://first.example/v1"}),
|
||||
promptkit.WithBackend(promptkit.Backend{ID: "second", Endpoint: "http://second.example/v1"}),
|
||||
promptkit.WithProfiles(
|
||||
promptkit.Profile{ID: "first-profile", BackendID: "first", Model: "model"},
|
||||
promptkit.Profile{ID: "second-profile", BackendID: "second", Model: "model"},
|
||||
),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine with accumulated registrations: %v", err)
|
||||
}
|
||||
for profileID, wantEndpoint := range map[string]string{
|
||||
"first-profile": "http://first.example/v1",
|
||||
"second-profile": "http://second.example/v1",
|
||||
} {
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "prompt", ProfileID: profileID,
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare %s: %v", profileID, err)
|
||||
}
|
||||
if prepared.EffectiveModelParams.Endpoint != wantEndpoint {
|
||||
t.Fatalf("profile %s endpoint=%q, want %q", profileID, prepared.EffectiveModelParams.Endpoint, wantEndpoint)
|
||||
}
|
||||
}
|
||||
|
||||
newEngine := func(endpoint string) *promptkit.Engine {
|
||||
t.Helper()
|
||||
value, err := promptkit.NewEngine(promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
|
||||
promptkit.WithBackend(promptkit.Backend{ID: "same-id", Endpoint: endpoint}),
|
||||
promptkit.WithProfiles(promptkit.Profile{ID: "profile", BackendID: "same-id", Model: "model"}),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct isolated engine: %v", err)
|
||||
}
|
||||
return value
|
||||
}
|
||||
firstEngine := newEngine("http://one.example/v1")
|
||||
secondEngine := newEngine("http://two.example/v1")
|
||||
for engine, wantEndpoint := range map[*promptkit.Engine]string{
|
||||
firstEngine: "http://one.example/v1",
|
||||
secondEngine: "http://two.example/v1",
|
||||
} {
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare isolated engine: %v", err)
|
||||
}
|
||||
if prepared.EffectiveModelParams.Endpoint != wantEndpoint {
|
||||
t.Fatalf("isolated engine endpoint=%q, want %q", prepared.EffectiveModelParams.Endpoint, wantEndpoint)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackendRegistrationRejectsInvalidAndDuplicateDefinitions(t *testing.T) {
|
||||
cycle := map[string]any{}
|
||||
cycle["self"] = cycle
|
||||
tests := []struct {
|
||||
name string
|
||||
backends []promptkit.Backend
|
||||
}{
|
||||
{name: "blank id", backends: []promptkit.Backend{{Endpoint: "http://example.test/v1"}}},
|
||||
{name: "invalid endpoint", backends: []promptkit.Backend{{ID: "custom", Endpoint: "ftp://example.test/v1"}}},
|
||||
{name: "invalid environment", backends: []promptkit.Backend{{ID: "custom", Endpoint: "http://example.test/v1", APIKeyEnv: "BAD-NAME"}}},
|
||||
{name: "reserved extra parameter", backends: []promptkit.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: map[string]any{"model": "override"}}}},
|
||||
{name: "cyclic extra parameter", backends: []promptkit.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: cycle}}},
|
||||
{name: "malformed JSON number", backends: []promptkit.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: map[string]any{"value": json.Number("01")}}}},
|
||||
{name: "duplicate consumer id", backends: []promptkit.Backend{
|
||||
{ID: " custom ", Endpoint: "http://one.example/v1"},
|
||||
{ID: "custom", Endpoint: "http://two.example/v1"},
|
||||
}},
|
||||
{name: "reserved built-in id", backends: []promptkit.Backend{{
|
||||
ID: promptkit.BackendOpenRouter, Endpoint: "http://replacement.example/v1",
|
||||
}}},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
options := []promptkit.Option{
|
||||
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
|
||||
}
|
||||
for _, backend := range tt.backends {
|
||||
options = append(options, promptkit.WithBackend(backend))
|
||||
}
|
||||
_, err := promptkit.NewEngine(promptkit.Config{}, options...)
|
||||
if !errors.Is(err, promptkit.ErrInvalidConfig) {
|
||||
t.Fatalf("expected ErrInvalidConfig, got %v", err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBackendExtraParamsAreDeeplyCopiedAtConstructionAndLookup(t *testing.T) {
|
||||
nested := map[string]any{"value": "original"}
|
||||
extraParams := map[string]any{"nested": nested}
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
|
||||
promptkit.WithBackend(promptkit.Backend{
|
||||
ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: extraParams,
|
||||
}),
|
||||
promptkit.WithProfiles(promptkit.Profile{ID: "profile", BackendID: "custom", Model: "model"}),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
nested["value"] = "mutated input"
|
||||
extraParams["later"] = true
|
||||
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||
if err != nil {
|
||||
t.Fatalf("first prepare: %v", err)
|
||||
}
|
||||
gotNested := prepared.EffectiveModelParams.ExtraParams["nested"].(map[string]any)
|
||||
if gotNested["value"] != "original" || prepared.EffectiveModelParams.ExtraParams["later"] != nil {
|
||||
t.Fatalf("backend retained caller mutations: %#v", prepared.EffectiveModelParams.ExtraParams)
|
||||
}
|
||||
gotNested["value"] = "mutated lookup"
|
||||
|
||||
prepared, err = engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||
if err != nil {
|
||||
t.Fatalf("second prepare: %v", err)
|
||||
}
|
||||
gotNested = prepared.EffectiveModelParams.ExtraParams["nested"].(map[string]any)
|
||||
if gotNested["value"] != "original" {
|
||||
t.Fatalf("backend retained lookup mutation: %#v", prepared.EffectiveModelParams.ExtraParams)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPreparedRunJSONTimingRoundTrips(t *testing.T) {
|
||||
start := time.Date(2026, time.July, 29, 12, 0, 0, 0, time.UTC)
|
||||
prepared := promptkit.PreparedRun{
|
||||
PromptID: "prompt",
|
||||
StartTime: start,
|
||||
EndTime: start.Add(1250 * time.Millisecond),
|
||||
DurationMS: 1250,
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(prepared)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal prepared run: %v", err)
|
||||
}
|
||||
var decoded promptkit.PreparedRun
|
||||
if err := json.Unmarshal(payload, &decoded); err != nil {
|
||||
t.Fatalf("unmarshal prepared run: %v", err)
|
||||
}
|
||||
if decoded.DurationMS != prepared.DurationMS ||
|
||||
!decoded.StartTime.Equal(prepared.StartTime) ||
|
||||
!decoded.EndTime.Equal(prepared.EndTime) {
|
||||
t.Fatalf("timing values did not round trip: got %#v, want %#v", decoded, prepared)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunResultJSONUsesMillisecondsAndRoundTrips(t *testing.T) {
|
||||
start := time.Date(2026, time.July, 29, 12, 0, 0, 0, time.UTC)
|
||||
result := promptkit.RunResult{
|
||||
RunID: "opaque-run-id",
|
||||
Artifact: promptkit.Artifact{Name: "output", ContentType: "text/plain", Body: []byte("ok")},
|
||||
SessionID: "session-123",
|
||||
StartTime: start,
|
||||
EndTime: start.Add(1500 * time.Millisecond),
|
||||
Duration: 1500 * time.Millisecond,
|
||||
}
|
||||
|
||||
payload, err := json.Marshal(result)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal run result: %v", err)
|
||||
}
|
||||
var object map[string]any
|
||||
if err := json.Unmarshal(payload, &object); err != nil {
|
||||
t.Fatalf("decode run result JSON: %v", err)
|
||||
}
|
||||
if got := object["duration_ms"]; got != float64(1500) {
|
||||
t.Fatalf("expected duration_ms=1500, got %#v in %s", got, payload)
|
||||
}
|
||||
if _, exists := object["duration"]; exists {
|
||||
t.Fatalf("unexpected nanosecond duration field in %s", payload)
|
||||
}
|
||||
if got := object["session_id"]; got != result.SessionID {
|
||||
t.Fatalf("expected session_id=%q, got %#v in %s", result.SessionID, got, payload)
|
||||
}
|
||||
artifact, ok := object["artifact"].(map[string]any)
|
||||
if !ok || artifact["content_type"] != "text/plain" {
|
||||
t.Fatalf("expected stable artifact JSON fields, got %#v", object["artifact"])
|
||||
}
|
||||
|
||||
var decoded promptkit.RunResult
|
||||
if err := json.Unmarshal(payload, &decoded); err != nil {
|
||||
t.Fatalf("unmarshal run result: %v", err)
|
||||
}
|
||||
if decoded.SessionID != result.SessionID ||
|
||||
decoded.Duration != result.Duration ||
|
||||
!decoded.StartTime.Equal(result.StartTime) ||
|
||||
!decoded.EndTime.Equal(result.EndTime) {
|
||||
t.Fatalf("timing values did not round trip: got %#v, want %#v", decoded, result)
|
||||
}
|
||||
|
||||
payload, err = json.Marshal(promptkit.RunResult{})
|
||||
if err != nil {
|
||||
t.Fatalf("marshal zero run result: %v", err)
|
||||
}
|
||||
for _, field := range []string{"session_id", "start_time", "end_time", "duration_ms"} {
|
||||
if strings.Contains(string(payload), `"`+field+`"`) {
|
||||
t.Fatalf("expected zero %s to be omitted, got %s", field, payload)
|
||||
}
|
||||
}
|
||||
var decodedEmpty promptkit.RunResult
|
||||
if err := json.Unmarshal(payload, &decodedEmpty); err != nil {
|
||||
t.Fatalf("unmarshal run result without session_id: %v", err)
|
||||
}
|
||||
if decodedEmpty.SessionID != "" {
|
||||
t.Fatalf("expected absent session_id to decode empty, got %q", decodedEmpty.SessionID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEngineValidationIsSinglePass(t *testing.T) {
|
||||
client := &fakeLLMClient{
|
||||
response: &promptkit.GenerateResponse{Content: "not-json"},
|
||||
}
|
||||
engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(client))
|
||||
|
||||
result, err := engine.Run(context.Background(), promptkit.RunRequest{
|
||||
PromptID: frameworkMarkdownSummaryPromptID,
|
||||
Inputs: map[string]promptkit.ArtifactRef{
|
||||
"transcript": promptkit.Inline("Rin opens the gate."),
|
||||
"glossary": promptkit.Inline("gate: A guarded passage."),
|
||||
},
|
||||
Validation: &promptkit.OutputContract{
|
||||
Format: promptkit.FormatJSON,
|
||||
ValidationMode: promptkit.ValidationJSON,
|
||||
RepairAttempts: 3,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("run with failed content validation: %v", err)
|
||||
}
|
||||
if result.Validation.Status != promptkit.ValidationFailed ||
|
||||
result.Validation.RepairAttempts != 0 {
|
||||
t.Fatalf("expected failed single-pass validation, got %#v", result.Validation)
|
||||
}
|
||||
if len(client.requests) != 1 {
|
||||
t.Fatalf("expected one model generation, got %d", len(client.requests))
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) {
|
||||
profile := promptkit.Profile{ID: "profile", Endpoint: "http://example.test/v1", Model: "model"}
|
||||
|
||||
t.Run("prompt source", func(t *testing.T) {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractPromptFS("first", "profile", "first"), "."),
|
||||
promptkit.WithPromptFS(contractPromptFS("second", "profile", "second"), "."),
|
||||
promptkit.WithProfiles(profile),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "second"})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare from last prompt source: %v", err)
|
||||
}
|
||||
if prepared.Messages[0].Content != "second" {
|
||||
t.Fatalf("expected last prompt source, got %#v", prepared.Messages)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("profile source", func(t *testing.T) {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
|
||||
promptkit.WithProfileFS(contractProfileFS("profile", "first-model"), "."),
|
||||
promptkit.WithProfileFS(contractProfileFS("profile", "second-model"), "."),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare from last profile source: %v", err)
|
||||
}
|
||||
if prepared.EffectiveModelParams.Model != "second-model" {
|
||||
t.Fatalf("expected last profile source, got %q", prepared.EffectiveModelParams.Model)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("in-memory profiles", func(t *testing.T) {
|
||||
first := profile
|
||||
first.Model = "first-model"
|
||||
second := profile
|
||||
second.Model = "second-model"
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
|
||||
promptkit.WithProfiles(first),
|
||||
promptkit.WithProfiles(second),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare from last in-memory profile option: %v", err)
|
||||
}
|
||||
if prepared.EffectiveModelParams.Model != "second-model" {
|
||||
t.Fatalf("expected last in-memory profiles, got %q", prepared.EffectiveModelParams.Model)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("schema source", func(t *testing.T) {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractSchemaPromptFS(), "."),
|
||||
promptkit.WithProfiles(profile),
|
||||
promptkit.WithSchemaFS(contractSchemaFS("first"), "."),
|
||||
promptkit.WithSchemaFS(contractSchemaFS("second"), "."),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "schema-prompt"})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare from last schema source: %v", err)
|
||||
}
|
||||
schema := prepared.StructuredOutput.JSONSchema.Schema.(map[string]any)
|
||||
if schema["title"] != "second" {
|
||||
t.Fatalf("expected last schema source, got %#v", schema)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("model client", func(t *testing.T) {
|
||||
var firstCalls, secondCalls atomic.Int64
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
|
||||
promptkit.WithProfiles(profile),
|
||||
promptkit.WithLLMClient(countingLLMClient{calls: &firstCalls}),
|
||||
promptkit.WithLLMClient(countingLLMClient{calls: &secondCalls}),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"}); err != nil {
|
||||
t.Fatalf("run with last model client: %v", err)
|
||||
}
|
||||
if firstCalls.Load() != 0 || secondCalls.Load() != 1 {
|
||||
t.Fatalf("expected only last client call, got first=%d second=%d", firstCalls.Load(), secondCalls.Load())
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("artifact reader", func(t *testing.T) {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractInputPromptFS(), "."),
|
||||
promptkit.WithProfiles(profile),
|
||||
promptkit.WithArtifactReader(fixedArtifactReader("first")),
|
||||
promptkit.WithArtifactReader(fixedArtifactReader("second")),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
|
||||
PromptID: "input-prompt",
|
||||
Inputs: map[string]promptkit.ArtifactRef{"input": promptkit.Inline("ignored")},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare with last artifact reader: %v", err)
|
||||
}
|
||||
if prepared.Messages[0].Content != "second" {
|
||||
t.Fatalf("expected last artifact reader, got %#v", prepared.Messages)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestEngineSupportsConcurrentPrepareAndRun(t *testing.T) {
|
||||
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
|
||||
promptkit.WithBackend(promptkit.Backend{
|
||||
ID: "concurrent",
|
||||
Endpoint: "http://example.test/v1",
|
||||
}),
|
||||
promptkit.WithProfiles(promptkit.Profile{
|
||||
ID: "profile",
|
||||
BackendID: "concurrent",
|
||||
Model: "model",
|
||||
}),
|
||||
promptkit.WithLLMClient(countingLLMClient{}),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("construct engine: %v", err)
|
||||
}
|
||||
|
||||
const calls = 40
|
||||
errs := make(chan error, calls)
|
||||
var wg sync.WaitGroup
|
||||
for i := 0; i < calls; i++ {
|
||||
wg.Add(1)
|
||||
go func(run bool) {
|
||||
defer wg.Done()
|
||||
request := promptkit.RunRequest{PromptID: "prompt"}
|
||||
if run {
|
||||
_, err := engine.Run(context.Background(), request)
|
||||
errs <- err
|
||||
return
|
||||
}
|
||||
_, err := engine.Prepare(context.Background(), request)
|
||||
errs <- err
|
||||
}(i%2 == 0)
|
||||
}
|
||||
wg.Wait()
|
||||
close(errs)
|
||||
for err := range errs {
|
||||
if err != nil {
|
||||
t.Fatalf("concurrent call failed: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
type countingLLMClient struct {
|
||||
calls *atomic.Int64
|
||||
}
|
||||
|
||||
func (c countingLLMClient) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
|
||||
if c.calls != nil {
|
||||
c.calls.Add(1)
|
||||
}
|
||||
return &promptkit.GenerateResponse{Content: "ok"}, nil
|
||||
}
|
||||
|
||||
type fixedArtifactReader string
|
||||
|
||||
func (r fixedArtifactReader) Read(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) {
|
||||
return &promptkit.Artifact{Body: []byte(r)}, nil
|
||||
}
|
||||
|
||||
func contractPromptFS(id, profileID, message string) fstest.MapFS {
|
||||
return fstest.MapFS{
|
||||
"prompt.yaml": &fstest.MapFile{Data: []byte(fmt.Sprintf(`id: %s
|
||||
version: "1"
|
||||
default_profile: %s
|
||||
messages:
|
||||
- role: user
|
||||
content: %q
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`, id, profileID, message))},
|
||||
}
|
||||
}
|
||||
|
||||
func contractInputPromptFS() fstest.MapFS {
|
||||
return fstest.MapFS{
|
||||
"prompt.yaml": &fstest.MapFile{Data: []byte(`id: input-prompt
|
||||
version: "1"
|
||||
default_profile: profile
|
||||
inputs:
|
||||
- name: input
|
||||
required: true
|
||||
messages:
|
||||
- role: user
|
||||
content: '{{input "input"}}'
|
||||
output:
|
||||
format: text
|
||||
validation_mode: none
|
||||
`)},
|
||||
}
|
||||
}
|
||||
|
||||
func contractProfileFS(id, model string) fstest.MapFS {
|
||||
return fstest.MapFS{
|
||||
"profile.yaml": &fstest.MapFile{Data: []byte(fmt.Sprintf(`id: %s
|
||||
endpoint: http://example.test/v1
|
||||
model: %s
|
||||
`, id, model))},
|
||||
}
|
||||
}
|
||||
|
||||
func contractSchemaPromptFS() fstest.MapFS {
|
||||
return fstest.MapFS{
|
||||
"prompt.yaml": &fstest.MapFile{Data: []byte(`id: schema-prompt
|
||||
version: "1"
|
||||
default_profile: profile
|
||||
messages:
|
||||
- role: user
|
||||
content: message
|
||||
output:
|
||||
format: json
|
||||
validation_mode: json_schema
|
||||
schema_path: schema.json
|
||||
`)},
|
||||
}
|
||||
}
|
||||
|
||||
func contractSchemaFS(title string) fstest.MapFS {
|
||||
return fstest.MapFS{
|
||||
"schema.json": &fstest.MapFile{Data: []byte(fmt.Sprintf(`{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"title": %q,
|
||||
"type": "object"
|
||||
}`, title))},
|
||||
}
|
||||
}
|
||||
643
types.go
643
types.go
@@ -5,302 +5,627 @@ import (
|
||||
"time"
|
||||
)
|
||||
|
||||
// ArtifactRefType defines how an artifact is referenced.
|
||||
// ArtifactRefType identifies how an [ArtifactRef] supplies content.
|
||||
type ArtifactRefType string
|
||||
|
||||
const (
|
||||
// ArtifactRefInline selects ArtifactRef.Body as the content.
|
||||
ArtifactRefInline ArtifactRefType = "inline"
|
||||
ArtifactRefFile ArtifactRefType = "file"
|
||||
// ArtifactRefFile selects the filesystem path in ArtifactRef.URI.
|
||||
ArtifactRefFile ArtifactRefType = "file"
|
||||
)
|
||||
|
||||
// OutputFormat defines the desired output format.
|
||||
// OutputFormat identifies the media format of generated output.
|
||||
// OutputFormat has a stable JSON string representation.
|
||||
type OutputFormat string
|
||||
|
||||
const (
|
||||
FormatText OutputFormat = "text"
|
||||
// FormatText identifies plain-text output.
|
||||
FormatText OutputFormat = "text"
|
||||
// FormatMarkdown identifies Markdown output.
|
||||
FormatMarkdown OutputFormat = "markdown"
|
||||
FormatJSON OutputFormat = "json"
|
||||
// FormatJSON identifies JSON output.
|
||||
FormatJSON OutputFormat = "json"
|
||||
)
|
||||
|
||||
// ValidationMode defines the output validation strategy.
|
||||
// ValidationMode identifies how generated output is checked.
|
||||
// ValidationMode has a stable JSON string representation.
|
||||
type ValidationMode string
|
||||
|
||||
const (
|
||||
ValidationNone ValidationMode = "none"
|
||||
ValidationBasic ValidationMode = "basic"
|
||||
ValidationJSON ValidationMode = "json"
|
||||
// ValidationNone skips content validation.
|
||||
ValidationNone ValidationMode = "none"
|
||||
// ValidationBasic requires non-empty output.
|
||||
ValidationBasic ValidationMode = "basic"
|
||||
// ValidationJSON requires syntactically valid JSON.
|
||||
ValidationJSON ValidationMode = "json"
|
||||
// ValidationJSONSchema requires JSON that satisfies OutputContract.SchemaPath.
|
||||
ValidationJSONSchema ValidationMode = "json_schema"
|
||||
)
|
||||
|
||||
// ValidationStatus defines the result of a validation check.
|
||||
// ValidationStatus identifies the completed state of an output check.
|
||||
// ValidationStatus has a stable JSON string representation.
|
||||
type ValidationStatus string
|
||||
|
||||
const (
|
||||
ValidationPassed ValidationStatus = "passed"
|
||||
ValidationFailed ValidationStatus = "failed"
|
||||
// ValidationPassed means the generated output satisfied its contract.
|
||||
ValidationPassed ValidationStatus = "passed"
|
||||
// ValidationFailed means validation completed and rejected the generated
|
||||
// output. Engine.Run returns this status in a result, not as an error.
|
||||
ValidationFailed ValidationStatus = "failed"
|
||||
// ValidationSkipped means ValidationNone selected no content check.
|
||||
ValidationSkipped ValidationStatus = "skipped"
|
||||
)
|
||||
|
||||
// CacheControlType defines provider cache behavior for prompt content.
|
||||
// CacheControlType identifies provider cache behavior for prompt content.
|
||||
// CacheControlType has a stable JSON string representation.
|
||||
type CacheControlType string
|
||||
|
||||
const (
|
||||
// CacheControlEphemeral requests provider-defined ephemeral caching.
|
||||
CacheControlEphemeral CacheControlType = "ephemeral"
|
||||
)
|
||||
|
||||
// StructuredOutputType identifies provider-level structured output modes.
|
||||
// StructuredOutputType has a stable JSON string representation.
|
||||
type StructuredOutputType string
|
||||
|
||||
const (
|
||||
// StructuredOutputJSONSchema supplies JSON Schema response constraints.
|
||||
StructuredOutputJSONSchema StructuredOutputType = "json_schema"
|
||||
)
|
||||
|
||||
// RunRequest represents a request to prepare or run a single prompt.
|
||||
// RunRequest selects one prompt execution. It has no stable JSON
|
||||
// representation.
|
||||
//
|
||||
// Prepare and Run copy the request's maps, pointers, and nested
|
||||
// JSON-compatible values before using them. The caller may mutate the request
|
||||
// after either method returns.
|
||||
type RunRequest struct {
|
||||
PromptID string
|
||||
// PromptID is the required non-empty prompt identifier.
|
||||
PromptID string
|
||||
// PromptVersion optionally selects one version of PromptID. When empty, the
|
||||
// prompt source must contain exactly one matching version.
|
||||
PromptVersion string
|
||||
ProfileID string
|
||||
APIKey string `json:"-"`
|
||||
Inputs map[string]ArtifactRef
|
||||
Vars map[string]string
|
||||
Execution *ExecutionTargetOverride
|
||||
Validation *OutputContract
|
||||
Metadata map[string]string
|
||||
// ProfileID selects an execution profile. When empty, the prompt's default
|
||||
// profile is used; if both are empty, the error matches ErrProfileRequired
|
||||
// and ErrInvalidRequest.
|
||||
ProfileID string
|
||||
// SessionID optionally supplies a direct per-run session identifier. A
|
||||
// nonblank value is trimmed and overrides the prompt definition's
|
||||
// session_id template. A blank value supplies no direct override. The
|
||||
// maximum is 256 Unicode code points after trimming. A direct value is
|
||||
// opaque consumer metadata, not a credential, and may be exposed in
|
||||
// prepared values, results, collaborator requests, provider requests, and
|
||||
// provider observability. Callers should use stable, non-sensitive
|
||||
// identifiers. An overlong direct value makes Prepare or Run return an
|
||||
// error matching ErrInvalidRequest.
|
||||
SessionID string
|
||||
// APIKey is a request-scoped direct credential. It takes precedence over
|
||||
// APIKeyEnv, is passed to the selected LLMClient, and is never included in
|
||||
// prepared values, results, hashes, JSON, String, or GoString output.
|
||||
APIKey string `json:"-"`
|
||||
// Inputs maps prompt input names to references. A nil or empty map is valid
|
||||
// only when the selected prompt and its templates require no inputs.
|
||||
Inputs map[string]ArtifactRef
|
||||
// Vars supplies Go-template data for messages and the session ID. Nil and
|
||||
// empty maps are equivalent.
|
||||
Vars map[string]string
|
||||
// Execution optionally overrides individual execution settings. Nil uses
|
||||
// the selected profile over its backend, when any, and framework defaults.
|
||||
Execution *ExecutionTargetOverride
|
||||
// Validation optionally replaces the prompt's complete output contract. It
|
||||
// does not merge individual fields. Nil uses the prompt contract.
|
||||
Validation *OutputContract
|
||||
}
|
||||
|
||||
// PreparedRun contains prepared prompt execution state. It does not include
|
||||
// resolved API key values, model output, validation results, or internal target
|
||||
// presence metadata.
|
||||
// presence metadata. PreparedRun has a stable JSON representation.
|
||||
//
|
||||
// All maps, slices, pointers, and schema values are caller-owned copies. JSON
|
||||
// timestamps use RFC 3339 and zero timing values are omitted. Hash formats are
|
||||
// opaque.
|
||||
type PreparedRun struct {
|
||||
PromptID string `json:"prompt_id"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
PromptHash string `json:"prompt_hash,omitempty"`
|
||||
SelectedProfileID string `json:"selected_profile_id"`
|
||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||
OutputContract OutputContract `json:"output_contract"`
|
||||
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||
Messages []RenderedMessage `json:"messages"`
|
||||
StartTime time.Time `json:"start_time,omitempty"`
|
||||
EndTime time.Time `json:"end_time,omitempty"`
|
||||
DurationMS int64 `json:"duration_ms,omitempty"`
|
||||
// PromptID is the selected prompt identifier.
|
||||
PromptID string `json:"prompt_id"`
|
||||
// PromptVersion is the selected prompt version.
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
// PromptHash is an opaque equality value for the selected definition.
|
||||
PromptHash string `json:"prompt_hash,omitempty"`
|
||||
// SelectedProfileID is the explicit request profile or prompt default that
|
||||
// supplied execution settings.
|
||||
SelectedProfileID string `json:"selected_profile_id"`
|
||||
// SelectedBackendID equals EffectiveModelParams.BackendID. It is empty for
|
||||
// an endpoint-only profile.
|
||||
SelectedBackendID string `json:"selected_backend_id,omitempty"`
|
||||
// EffectiveModelParams contains framework defaults overlaid by the selected
|
||||
// backend, profile, and then request overrides. It excludes resolved API-key
|
||||
// values.
|
||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||
// OutputContract is the complete effective output contract.
|
||||
OutputContract OutputContract `json:"output_contract"`
|
||||
// StructuredOutput is non-nil for JSON Schema validation and contains the
|
||||
// provider-facing response constraint passed to an LLM client.
|
||||
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
||||
// InputHashes maps every supplied input name to its opaque artifact hash.
|
||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||
// SessionID is the effective direct or rendered session identifier, if any.
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
// RenderedPromptHash is an opaque equality value for SessionID and Messages.
|
||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||
// Messages are the rendered messages that Run passes to the LLM client.
|
||||
Messages []RenderedMessage `json:"messages"`
|
||||
// StartTime is the UTC time at which preparation began.
|
||||
StartTime time.Time `json:"start_time,omitempty"`
|
||||
// EndTime is the UTC time at which preparation completed.
|
||||
EndTime time.Time `json:"end_time,omitempty"`
|
||||
// DurationMS is preparation elapsed time in integer milliseconds. JSON uses
|
||||
// duration_ms and omits a zero value.
|
||||
DurationMS int64 `json:"duration_ms,omitempty"`
|
||||
}
|
||||
|
||||
// RunResult contains generated output, validation state, and run metadata.
|
||||
// RunResult has a stable JSON representation and round-trips its Duration
|
||||
// through the duration_ms JSON field.
|
||||
//
|
||||
// All maps, slices, and nested values are caller-owned copies. JSON timestamps
|
||||
// use RFC 3339 and zero timing values are omitted. Run IDs and hash formats are
|
||||
// opaque.
|
||||
type RunResult struct {
|
||||
RunID string `json:"run_id"`
|
||||
Artifact Artifact `json:"artifact"`
|
||||
RawOutput string `json:"raw_output"`
|
||||
Validation ValidationResult `json:"validation"`
|
||||
PromptID string `json:"prompt_id"`
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
PromptHash string `json:"prompt_hash,omitempty"`
|
||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||
SelectedProfileID string `json:"selected_profile_id"`
|
||||
ModelName string `json:"model_name"`
|
||||
Endpoint string `json:"endpoint"`
|
||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||
Usage TokenUsage `json:"usage"`
|
||||
StartTime time.Time `json:"start_time,omitempty"`
|
||||
EndTime time.Time `json:"end_time,omitempty"`
|
||||
Duration time.Duration `json:"duration,omitempty"`
|
||||
// RunID is an opaque identifier for this invocation.
|
||||
RunID string `json:"run_id"`
|
||||
// Artifact contains the generated output and derived metadata.
|
||||
Artifact Artifact `json:"artifact"`
|
||||
// RawOutput is the exact generated content before artifact classification
|
||||
// and validation.
|
||||
RawOutput string `json:"raw_output"`
|
||||
// Validation records the completed content check.
|
||||
Validation ValidationResult `json:"validation"`
|
||||
// PromptID is the selected prompt identifier.
|
||||
PromptID string `json:"prompt_id"`
|
||||
// PromptVersion is the selected prompt version.
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
// PromptHash is the same opaque definition equality value exposed by
|
||||
// PreparedRun.
|
||||
PromptHash string `json:"prompt_hash,omitempty"`
|
||||
// SessionID is the effective direct or rendered session identifier, if any.
|
||||
// JSON omits an empty value.
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
// RenderedPromptHash is the same opaque rendered-prompt equality value
|
||||
// computed during preparation.
|
||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||
// SelectedProfileID identifies the profile used for execution.
|
||||
SelectedProfileID string `json:"selected_profile_id"`
|
||||
// SelectedBackendID equals EffectiveModelParams.BackendID. It is empty for
|
||||
// an endpoint-only profile.
|
||||
SelectedBackendID string `json:"selected_backend_id,omitempty"`
|
||||
// ModelName is the effective model name and equals
|
||||
// EffectiveModelParams.Model.
|
||||
ModelName string `json:"model_name"`
|
||||
// Endpoint is the effective base endpoint and equals
|
||||
// EffectiveModelParams.Endpoint.
|
||||
Endpoint string `json:"endpoint"`
|
||||
// EffectiveModelParams contains the settings supplied to the LLM client,
|
||||
// excluding resolved API-key values.
|
||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||
// InputHashes are the opaque input equality values computed during
|
||||
// preparation.
|
||||
InputHashes map[string]string `json:"input_hashes,omitempty"`
|
||||
// Usage is the token accounting reported by the LLM client.
|
||||
Usage TokenUsage `json:"usage"`
|
||||
// StartTime is the UTC time immediately before preparation begins.
|
||||
StartTime time.Time `json:"start_time,omitempty"`
|
||||
// EndTime is the UTC time after generation and validation complete.
|
||||
EndTime time.Time `json:"end_time,omitempty"`
|
||||
// Duration covers preparation, generation, and validation. JSON represents
|
||||
// it as integer milliseconds in duration_ms and omits a zero value.
|
||||
Duration time.Duration `json:"-"`
|
||||
}
|
||||
|
||||
// ArtifactRef represents a reference to prompt input content.
|
||||
// ArtifactRef identifies prompt input content. It has no stable JSON
|
||||
// representation. Prefer [File], [Inline], or [InlineWithURI] to construct one.
|
||||
type ArtifactRef struct {
|
||||
// Type must be ArtifactRefInline or ArtifactRefFile.
|
||||
Type ArtifactRefType
|
||||
URI string
|
||||
// URI is the file path for ArtifactRefFile and optional provenance metadata
|
||||
// for ArtifactRefInline.
|
||||
URI string
|
||||
// Body is the content for ArtifactRefInline and is ignored for
|
||||
// ArtifactRefFile.
|
||||
Body string
|
||||
}
|
||||
|
||||
// Artifact represents loaded artifact content.
|
||||
// Artifact represents loaded or generated content and has a stable JSON
|
||||
// representation. Body uses encoding/json's base64 representation for []byte.
|
||||
type Artifact struct {
|
||||
Name string
|
||||
ContentType string
|
||||
Body []byte
|
||||
URI string
|
||||
Size int64
|
||||
Hash string
|
||||
// Name is artifact metadata. During input preparation the engine fills an
|
||||
// empty reader-supplied name with the request input-map key.
|
||||
Name string `json:"name"`
|
||||
// ContentType is the media type reported by the reader or derived for
|
||||
// generated output.
|
||||
ContentType string `json:"content_type"`
|
||||
// Body is the artifact content. Engine boundaries copy this slice.
|
||||
Body []byte `json:"body"`
|
||||
// URI is optional source or result provenance metadata.
|
||||
URI string `json:"uri"`
|
||||
// Size is content-size metadata in bytes.
|
||||
Size int64 `json:"size"`
|
||||
// Hash is an opaque content equality value when the producing reader
|
||||
// supplies one. Its format and algorithm are not API contracts.
|
||||
Hash string `json:"hash"`
|
||||
}
|
||||
|
||||
// ArtifactReader resolves a prompt input reference into its content.
|
||||
//
|
||||
// Readers are responsible for supplying artifact metadata. The engine assigns
|
||||
// an input-map name only when the returned artifact name is empty.
|
||||
// Read may be called concurrently. It must honor ctx cancellation to make
|
||||
// Prepare and Run responsive to cancellation. The engine passes a copied ref
|
||||
// and immediately copies the returned Artifact.Body; it does not retain either
|
||||
// value. Readers supply artifact metadata, and the engine assigns an input-map
|
||||
// name only when the returned artifact name is empty.
|
||||
//
|
||||
// An injected reader owns any application-specific path containment,
|
||||
// authorization, content-size, and content-type policy. It must protect
|
||||
// sensitive references and bodies in its logging and in any copies it retains.
|
||||
// It may reuse or mutate the returned artifact and body after Read returns.
|
||||
//
|
||||
// Returning a non-nil error makes the engine return an error matching
|
||||
// ErrArtifactLoad while preserving the reader error through errors.Is.
|
||||
// Returning a nil artifact with a nil error also produces ErrArtifactLoad.
|
||||
type ArtifactReader interface {
|
||||
Read(context.Context, ArtifactRef) (*Artifact, error)
|
||||
}
|
||||
|
||||
// ExecutionTarget represents effective model runtime settings.
|
||||
// ExecutionTarget represents effective model runtime settings and has a stable
|
||||
// JSON representation. It never exposes a resolved API-key value.
|
||||
type ExecutionTarget struct {
|
||||
Endpoint string `json:"endpoint"`
|
||||
Model string `json:"model"`
|
||||
Temperature float64 `json:"temperature"`
|
||||
MaxTokens int `json:"max_tokens"`
|
||||
TopP float64 `json:"top_p"`
|
||||
TimeoutSeconds int `json:"timeout_seconds"`
|
||||
ServiceTier string `json:"service_tier"`
|
||||
ReasoningEffort string `json:"reasoning_effort"`
|
||||
APIKeyEnv string `json:"api_key_env"`
|
||||
ExtraParams map[string]any `json:"extra_params"`
|
||||
// BackendID is the effective routing identity selected by the profile. It
|
||||
// remains unchanged when a profile or request overrides Endpoint and is
|
||||
// empty for endpoint-only profiles. It is supplied to injected LLMClient
|
||||
// implementations as part of the effective target.
|
||||
BackendID string `json:"backend_id,omitempty"`
|
||||
// Endpoint is the model-provider base URL.
|
||||
Endpoint string `json:"endpoint"`
|
||||
// Model is the provider model identifier.
|
||||
Model string `json:"model"`
|
||||
// Temperature is the effective sampling temperature from 0 through 2.
|
||||
Temperature float64 `json:"temperature"`
|
||||
// MaxTokens is the non-negative effective output-token limit. Zero leaves
|
||||
// the limit unspecified to compatible providers unless it was an explicit
|
||||
// request override.
|
||||
MaxTokens int `json:"max_tokens"`
|
||||
// TopP is the effective nucleus-sampling value from 0 through 1.
|
||||
TopP float64 `json:"top_p"`
|
||||
// TimeoutSeconds is the non-negative per-generation deadline. Zero disables
|
||||
// this deadline without disabling caller cancellation or the transport cap.
|
||||
TimeoutSeconds int `json:"timeout_seconds"`
|
||||
// ServiceTier is an optional provider-specific request tier.
|
||||
ServiceTier string `json:"service_tier"`
|
||||
// ReasoningEffort is the effective opaque provider-specific reasoning
|
||||
// setting. An empty value instructs model clients to omit reasoning.
|
||||
ReasoningEffort string `json:"reasoning_effort"`
|
||||
// APIKeyEnv is an environment-variable name, not its credential value.
|
||||
APIKeyEnv string `json:"api_key_env"`
|
||||
// ExtraParams contains copied JSON-compatible provider parameters.
|
||||
ExtraParams map[string]any `json:"extra_params"`
|
||||
}
|
||||
|
||||
// ExecutionTargetOverride represents per-request runtime setting overrides.
|
||||
// ExecutionTargetOverride represents per-request runtime setting overrides and
|
||||
// has no stable JSON representation.
|
||||
//
|
||||
// Non-empty string fields replace profile and backend values. Non-nil pointer
|
||||
// fields replace profile values and preserve explicit zero or empty values. A
|
||||
// non-empty ExtraParams map replaces the complete profile or backend map
|
||||
// rather than merging keys. Empty string fields, nil pointers, and a nil or
|
||||
// empty ExtraParams map inherit the selected profile over its backend, when
|
||||
// any, and framework defaults.
|
||||
type ExecutionTargetOverride struct {
|
||||
Endpoint string
|
||||
Model string
|
||||
Temperature *float64
|
||||
MaxTokens *int
|
||||
TopP *float64
|
||||
TimeoutSeconds *int
|
||||
ServiceTier string
|
||||
ReasoningEffort string
|
||||
APIKeyEnv string
|
||||
ExtraParams map[string]any
|
||||
// Endpoint replaces the profile or backend endpoint when non-empty without
|
||||
// changing the effective BackendID.
|
||||
Endpoint string
|
||||
// Model replaces the profile model when non-empty.
|
||||
Model string
|
||||
// Temperature, when non-nil, must point to a value from 0 through 2.
|
||||
Temperature *float64
|
||||
// MaxTokens, when non-nil, must point to a non-negative value.
|
||||
MaxTokens *int
|
||||
// TopP, when non-nil, must point to a value from 0 through 1.
|
||||
TopP *float64
|
||||
// TimeoutSeconds, when non-nil, must point to a non-negative value. A
|
||||
// pointed-to zero disables the per-generation deadline.
|
||||
TimeoutSeconds *int
|
||||
// ServiceTier replaces the profile value when non-blank.
|
||||
ServiceTier string
|
||||
// ReasoningEffort controls the per-run reasoning setting. Nil inherits the
|
||||
// profile value. A pointer to a non-blank string trims and replaces the
|
||||
// profile value. A pointer to an empty or whitespace-only string clears the
|
||||
// inherited value and disables reasoning for this run. Non-blank values
|
||||
// are opaque and are not validated against a fixed vocabulary.
|
||||
ReasoningEffort *string
|
||||
// APIKeyEnv replaces the profile or backend environment-variable name when
|
||||
// non-blank. A direct RunRequest.APIKey still takes precedence over
|
||||
// environment lookup.
|
||||
APIKeyEnv string
|
||||
// ExtraParams, when non-empty, replaces the complete profile or backend map.
|
||||
// Values must be JSON-compatible: nil, booleans, finite numbers, strings,
|
||||
// arrays or slices, and maps with non-empty string keys. Cycles are invalid.
|
||||
ExtraParams map[string]any
|
||||
}
|
||||
|
||||
// Profile is an in-memory execution profile for library consumers.
|
||||
//
|
||||
// It is equivalent to a loaded profile file after validation. Raw API keys do
|
||||
// not belong in profiles; use APIKeyRequired to require callers to provide
|
||||
// RunRequest.APIKey for each request, or use profile YAML api_key_env with file
|
||||
// and FS profile sources.
|
||||
// not belong in profiles; use APIKeyRequired to require callers to provide a
|
||||
// RunRequest.APIKey or explicit request ExecutionTargetOverride.APIKeyEnv, or
|
||||
// use profile YAML api_key_env with file and FS profile sources. Profile has no
|
||||
// stable JSON representation.
|
||||
//
|
||||
// WithProfiles validates and copies Profile values during NewEngine. Numeric
|
||||
// zero, blank strings, and an empty ExtraParams map inherit framework defaults;
|
||||
// use ExecutionTargetOverride pointer fields to request explicit numeric zero.
|
||||
type Profile struct {
|
||||
ID string
|
||||
Endpoint string
|
||||
Model string
|
||||
Temperature float64
|
||||
MaxTokens int
|
||||
TopP float64
|
||||
TimeoutSeconds int
|
||||
ServiceTier string
|
||||
// ID is the required non-blank profile identifier. WithProfiles trims it.
|
||||
ID string
|
||||
// BackendID optionally selects an engine backend. WithProfiles trims it.
|
||||
// Backend membership is checked when a request selects the profile; an
|
||||
// unknown ID makes preparation fail with ErrProfileLoad.
|
||||
BackendID string
|
||||
// Endpoint is the model-provider base URL. It is required only when
|
||||
// BackendID is blank and otherwise overrides the backend endpoint when
|
||||
// non-blank.
|
||||
Endpoint string
|
||||
// Model is the required non-blank provider model identifier.
|
||||
Model string
|
||||
// Temperature is from 0 through 2. Zero inherits the framework default.
|
||||
Temperature float64
|
||||
// MaxTokens is non-negative. Zero inherits the framework default.
|
||||
MaxTokens int
|
||||
// TopP is from 0 through 1. Zero inherits the framework default rather than
|
||||
// selecting an explicit zero.
|
||||
TopP float64
|
||||
// TimeoutSeconds is non-negative. Zero inherits the framework default.
|
||||
TimeoutSeconds int
|
||||
// ServiceTier is optional; a blank value inherits the framework default.
|
||||
ServiceTier string
|
||||
// ReasoningEffort is optional; a blank value inherits the framework
|
||||
// default.
|
||||
ReasoningEffort string
|
||||
APIKeyRequired bool
|
||||
ExtraParams map[string]any
|
||||
// APIKeyRequired clears a backend's inherited API-key environment name and
|
||||
// requires a non-blank RunRequest.APIKey unless the request explicitly
|
||||
// supplies ExecutionTargetOverride.APIKeyEnv. It does not store a credential.
|
||||
APIKeyRequired bool
|
||||
// ExtraParams contains provider-specific JSON-compatible values. An empty
|
||||
// map inherits backend request defaults, when any. WithProfiles validates
|
||||
// and deeply copies it during NewEngine.
|
||||
ExtraParams map[string]any
|
||||
}
|
||||
|
||||
// OpenAICompatibleProfileConfig configures an OpenAI-compatible in-memory
|
||||
// profile.
|
||||
//
|
||||
// It contains ordinary profile fields for OpenAI-compatible chat-completions
|
||||
// endpoints. APIKeyRequired is satisfied by RunRequest.APIKey. Raw API keys do
|
||||
// not belong in this config.
|
||||
// endpoints. APIKeyRequired follows Profile.APIKeyRequired. Raw API keys do not
|
||||
// belong in this config. OpenAICompatibleProfileConfig has no stable JSON
|
||||
// representation and is not validated until its resulting Profile is supplied
|
||||
// through WithProfiles to NewEngine.
|
||||
type OpenAICompatibleProfileConfig struct {
|
||||
ID string
|
||||
Endpoint string
|
||||
Model string
|
||||
APIKeyRequired bool
|
||||
Temperature float64
|
||||
MaxTokens int
|
||||
TopP float64
|
||||
TimeoutSeconds int
|
||||
ServiceTier string
|
||||
// ID becomes Profile.ID.
|
||||
ID string
|
||||
// BackendID becomes Profile.BackendID.
|
||||
BackendID string
|
||||
// Endpoint becomes Profile.Endpoint.
|
||||
Endpoint string
|
||||
// Model becomes Profile.Model.
|
||||
Model string
|
||||
// APIKeyRequired becomes Profile.APIKeyRequired.
|
||||
APIKeyRequired bool
|
||||
// Temperature becomes Profile.Temperature.
|
||||
Temperature float64
|
||||
// MaxTokens becomes Profile.MaxTokens.
|
||||
MaxTokens int
|
||||
// TopP becomes Profile.TopP.
|
||||
TopP float64
|
||||
// TimeoutSeconds becomes Profile.TimeoutSeconds.
|
||||
TimeoutSeconds int
|
||||
// ServiceTier becomes Profile.ServiceTier.
|
||||
ServiceTier string
|
||||
// ReasoningEffort becomes Profile.ReasoningEffort.
|
||||
ReasoningEffort string
|
||||
ExtraParams map[string]any
|
||||
// ExtraParams becomes a shallow-copied Profile.ExtraParams map. NewEngine
|
||||
// performs validation and a deep copy when WithProfiles applies the result.
|
||||
ExtraParams map[string]any
|
||||
}
|
||||
|
||||
// ExecutionTargetPresence tracks which numeric runtime settings were explicit
|
||||
// request overrides.
|
||||
// request overrides, including explicit zero values. It has a stable JSON
|
||||
// representation and is supplied to injected LLM clients so they can preserve
|
||||
// omission semantics.
|
||||
type ExecutionTargetPresence struct {
|
||||
Temperature bool
|
||||
MaxTokens bool
|
||||
TopP bool
|
||||
TimeoutSeconds bool
|
||||
// Temperature reports a non-nil ExecutionTargetOverride.Temperature.
|
||||
Temperature bool `json:"temperature"`
|
||||
// MaxTokens reports a non-nil ExecutionTargetOverride.MaxTokens.
|
||||
MaxTokens bool `json:"max_tokens"`
|
||||
// TopP reports a non-nil ExecutionTargetOverride.TopP.
|
||||
TopP bool `json:"top_p"`
|
||||
// TimeoutSeconds reports a non-nil ExecutionTargetOverride.TimeoutSeconds.
|
||||
TimeoutSeconds bool `json:"timeout_seconds"`
|
||||
}
|
||||
|
||||
// OutputContract defines output and validation requirements.
|
||||
// OutputContract defines output and validation requirements and has a stable
|
||||
// JSON representation.
|
||||
//
|
||||
// A non-nil RunRequest.Validation replaces the complete prompt contract. It
|
||||
// does not merge fields. The public Engine validates generated output once and
|
||||
// does not install an output repairer.
|
||||
type OutputContract struct {
|
||||
Format OutputFormat `json:"format"`
|
||||
// Format selects generated artifact metadata. An empty effective value
|
||||
// defaults to FormatText.
|
||||
Format OutputFormat `json:"format"`
|
||||
// ValidationMode selects the content check. Use one of the declared
|
||||
// ValidationMode constants.
|
||||
ValidationMode ValidationMode `json:"validation_mode"`
|
||||
SchemaPath string `json:"schema_path"`
|
||||
RepairAttempts int `json:"repair_attempts"`
|
||||
// SchemaPath is required when ValidationMode is ValidationJSONSchema and is
|
||||
// ignored by other modes.
|
||||
SchemaPath string `json:"schema_path"`
|
||||
// RepairAttempts is a requested repair limit. A non-positive value requests
|
||||
// no repairs. The public Engine performs no repairs even when this value is
|
||||
// positive, so its runs report zero attempts used.
|
||||
RepairAttempts int `json:"repair_attempts"`
|
||||
}
|
||||
|
||||
// ValidationResult represents output validation state.
|
||||
// ValidationResult represents a completed output check and has a stable JSON
|
||||
// representation. An operational inability to perform validation is returned
|
||||
// as ErrValidation instead of a ValidationResult.
|
||||
type ValidationResult struct {
|
||||
Status ValidationStatus `json:"status"`
|
||||
Mode ValidationMode `json:"mode"`
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
SchemaPath string `json:"schema_path,omitempty"`
|
||||
RepairAttempts int `json:"repair_attempts"`
|
||||
IsValid bool `json:"is_valid"`
|
||||
// Status is Passed, Failed, or Skipped.
|
||||
Status ValidationStatus `json:"status"`
|
||||
// Mode is the effective validation mode.
|
||||
Mode ValidationMode `json:"mode"`
|
||||
// Errors contains validation diagnostics when Status is ValidationFailed.
|
||||
Errors []string `json:"errors,omitempty"`
|
||||
// SchemaPath is the effective schema path for JSON Schema validation.
|
||||
SchemaPath string `json:"schema_path,omitempty"`
|
||||
// RepairAttempts is the number of repairs actually attempted. It is always
|
||||
// zero for the public Engine.
|
||||
RepairAttempts int `json:"repair_attempts"`
|
||||
// IsValid is true for ValidationPassed and ValidationSkipped and false for
|
||||
// ValidationFailed.
|
||||
IsValid bool `json:"is_valid"`
|
||||
}
|
||||
|
||||
// TokenUsage tracks token consumption.
|
||||
// TokenUsage contains model-client token accounting and has a stable JSON
|
||||
// representation. Promptkit preserves values reported by the client and does
|
||||
// not derive or reconcile them.
|
||||
type TokenUsage struct {
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
// PromptTokens is the reported input-token count.
|
||||
PromptTokens int `json:"prompt_tokens"`
|
||||
// CompletionTokens is the reported generated-token count.
|
||||
CompletionTokens int `json:"completion_tokens"`
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
CachedTokens int `json:"cached_tokens"`
|
||||
// TotalTokens is the reported total-token count.
|
||||
TotalTokens int `json:"total_tokens"`
|
||||
// CachedTokens is the reported cached-input-token count.
|
||||
CachedTokens int `json:"cached_tokens"`
|
||||
// CacheWriteTokens is the reported cache-write-token count.
|
||||
CacheWriteTokens int `json:"cache_write_tokens"`
|
||||
}
|
||||
|
||||
// RenderedPrompt is the fully rendered prompt passed to an LLM client.
|
||||
// RenderedPrompt is the fully rendered prompt passed to an LLM client and has
|
||||
// a stable JSON representation.
|
||||
type RenderedPrompt struct {
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
Messages []RenderedMessage `json:"messages"`
|
||||
// SessionID is the optional effective direct or rendered session
|
||||
// identifier supplied to the model client.
|
||||
SessionID string `json:"session_id,omitempty"`
|
||||
// Messages contains rendered messages in definition order.
|
||||
Messages []RenderedMessage `json:"messages"`
|
||||
}
|
||||
|
||||
// RenderedMessage is a rendered chat message.
|
||||
// RenderedMessage is a rendered chat message and has a stable JSON
|
||||
// representation.
|
||||
type RenderedMessage struct {
|
||||
Role string `json:"role"`
|
||||
Content string `json:"content"`
|
||||
// Role is the definition-supplied chat role.
|
||||
Role string `json:"role"`
|
||||
// Content is the rendered message text.
|
||||
Content string `json:"content"`
|
||||
// CacheControl is optional provider cache metadata.
|
||||
CacheControl *CacheControl `json:"cache_control,omitempty"`
|
||||
}
|
||||
|
||||
// CacheControl describes provider cache metadata attached to prompt content.
|
||||
// CacheControl describes provider cache metadata attached to prompt content
|
||||
// and has a stable JSON representation.
|
||||
type CacheControl struct {
|
||||
// Type identifies the cache behavior.
|
||||
Type CacheControlType `json:"type"`
|
||||
TTL string `json:"ttl,omitempty"`
|
||||
// TTL is an optional provider cache lifetime.
|
||||
TTL string `json:"ttl,omitempty"`
|
||||
}
|
||||
|
||||
// StructuredOutputSpec describes provider-level structured output.
|
||||
// StructuredOutputSpec describes provider-level structured output and has a
|
||||
// stable JSON representation.
|
||||
type StructuredOutputSpec struct {
|
||||
Type StructuredOutputType `json:"type"`
|
||||
// Type identifies the structured-output mechanism.
|
||||
Type StructuredOutputType `json:"type"`
|
||||
// JSONSchema contains constraints when Type is StructuredOutputJSONSchema.
|
||||
JSONSchema *StructuredOutputJSONSpec `json:"json_schema,omitempty"`
|
||||
}
|
||||
|
||||
// StructuredOutputJSONSpec contains JSON Schema output constraints.
|
||||
// StructuredOutputJSONSpec contains provider-facing JSON Schema output
|
||||
// constraints and has a stable JSON representation.
|
||||
type StructuredOutputJSONSpec struct {
|
||||
Name string `json:"name"`
|
||||
Strict bool `json:"strict"`
|
||||
Schema any `json:"schema"`
|
||||
// Name is the provider-facing schema name.
|
||||
Name string `json:"name"`
|
||||
// Strict requests strict provider enforcement of Schema.
|
||||
Strict bool `json:"strict"`
|
||||
// Schema is a caller-owned copy of the loaded JSON Schema document.
|
||||
Schema any `json:"schema"`
|
||||
}
|
||||
|
||||
// LLMClient executes rendered prompts for Engine.Run.
|
||||
// LLMClient executes rendered prompts for [Engine.Run].
|
||||
//
|
||||
// Generate may be called concurrently. It must honor context cancellation to
|
||||
// make Run responsive to cancellation. The request and all nested maps,
|
||||
// slices, and pointers are client-owned copies and may be mutated or retained
|
||||
// without affecting engine state.
|
||||
//
|
||||
// Generate receives rendered messages and may receive a direct API key. A
|
||||
// client must protect those values and any raw output in its logging, storage,
|
||||
// and retained copies. It is responsible for the cancellation behavior of any
|
||||
// work it starts and for synchronizing access to retained or shared data.
|
||||
//
|
||||
// A returned error makes Run return ErrLLMGenerate while preserving the client
|
||||
// error through errors.Is. A nil response with a nil error also produces
|
||||
// ErrLLMGenerate. Promptkit copies the non-nil response before returning from
|
||||
// Run.
|
||||
type LLMClient interface {
|
||||
Generate(context.Context, GenerateRequest) (*GenerateResponse, error)
|
||||
}
|
||||
|
||||
// GenerateRequest is passed to an injected LLM client.
|
||||
// GenerateRequest is passed to an injected LLM client and has a stable JSON
|
||||
// representation. Its String and GoString methods omit rendered content and
|
||||
// direct credentials.
|
||||
type GenerateRequest struct {
|
||||
Prompt RenderedPrompt `json:"prompt"`
|
||||
Target ExecutionTarget `json:"target"`
|
||||
TargetPresence ExecutionTargetPresence `json:"target_presence"`
|
||||
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
||||
APIKey string `json:"-"`
|
||||
// Prompt contains the rendered session ID and messages.
|
||||
Prompt RenderedPrompt `json:"prompt"`
|
||||
// Target contains effective model settings without the direct API key.
|
||||
Target ExecutionTarget `json:"target"`
|
||||
// TargetPresence distinguishes inherited numeric zeros from explicit
|
||||
// request overrides.
|
||||
TargetPresence ExecutionTargetPresence `json:"target_presence"`
|
||||
// StructuredOutput contains provider response constraints when requested.
|
||||
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
||||
// APIKey is the direct request-scoped credential, if any. It is excluded
|
||||
// from JSON, String, and GoString output.
|
||||
APIKey string `json:"-"`
|
||||
}
|
||||
|
||||
// GenerateResponse is returned by an injected LLM client.
|
||||
// GenerateResponse is returned by an injected LLM client and has a stable JSON
|
||||
// representation.
|
||||
type GenerateResponse struct {
|
||||
Content string `json:"content"`
|
||||
Usage TokenUsage `json:"usage"`
|
||||
// Content is the generated output. It must be non-empty when using the
|
||||
// built-in client; injected clients may return empty content for Promptkit
|
||||
// validation to classify.
|
||||
Content string `json:"content"`
|
||||
// Usage is the client's token accounting.
|
||||
Usage TokenUsage `json:"usage"`
|
||||
}
|
||||
|
||||
// File returns a file-backed artifact reference.
|
||||
// File returns a file-backed artifact reference whose URI is path.
|
||||
//
|
||||
// The default artifact reader opens path as a caller-selected operating-system
|
||||
// path without restricting it to an application root or imposing a size limit.
|
||||
// Applications accepting untrusted paths must validate them before calling
|
||||
// Promptkit or use [WithArtifactReader] to enforce application policy.
|
||||
func File(path string) ArtifactRef {
|
||||
return ArtifactRef{Type: ArtifactRefFile, URI: path}
|
||||
}
|
||||
|
||||
// Inline returns an inline artifact reference.
|
||||
// Inline returns an inline artifact reference whose Body is body and whose URI
|
||||
// is empty.
|
||||
func Inline(body string) ArtifactRef {
|
||||
return ArtifactRef{Type: ArtifactRefInline, Body: body}
|
||||
}
|
||||
|
||||
// InlineWithURI returns an inline artifact reference with URI metadata.
|
||||
// InlineWithURI returns an inline artifact reference with body content and uri
|
||||
// provenance metadata.
|
||||
func InlineWithURI(uri string, body string) ArtifactRef {
|
||||
return ArtifactRef{Type: ArtifactRefInline, URI: uri, Body: body}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user