eric/promptkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Cut over public engine construction to ConfigKit sources

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-30 03:18:49 +00:00
parent 17b6c7bca5
commit ba9c972faa
37 changed files with 1653 additions and 1423 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
appended_messages_contract_test.go
View File

@@ -8,6 +8,7 @@ import (
"testing"
"gitea.maximumdirect.net/eric/promptkit"
"gitea.maximumdirect.net/eric/promptkit/testfixture"
)
func TestAppendedMessageRoleConstantsAreStrings(t *testing.T) {
@@ -29,10 +30,10 @@ func TestAppendedMessagesComposeFrozenEffectivePrompt(t *testing.T) {
{Role: promptkit.RoleUser, Content: "corrective request", CacheControl: cacheControl},
}
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "output"}}
engine, err := promptkit.NewEngine(
promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "configured prompt"), "."),
promptkit.WithProfiles(promptkit.Profile{ID: "profile", Endpoint: "http://example.test/v1", Model: "model"}),
engine, err := testfixture.NewEngine(
testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "configured prompt"), "."),
testfixture.WithProfiles(testfixture.Profile{ID: "profile", Endpoint: "http://example.test/v1", Model: "model"}),
promptkit.WithLLMClient(client),
)
if err != nil {
@@ -101,14 +102,9 @@ func TestAppendedMessagesComposeFrozenEffectivePrompt(t *testing.T) {
}
}
func TestInvalidAppendedMessagesFailBeforeSourceOrModelWork(t *testing.T) {
promptSource := &inspectionCountingFS{}
func TestInvalidAppendedMessagesFailBeforeModelWork(t *testing.T) {
var modelCalls atomic.Int64
engine, err := promptkit.NewEngine(
promptkit.Config{},
promptkit.WithPromptFS(promptSource, "."),
promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}),
)
engine, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithLLMClient(countingLLMClient{calls: &modelCalls}))
if err != nil {
t.Fatalf("construct engine: %v", err)
}
@@ -136,9 +132,6 @@ func TestInvalidAppendedMessagesFailBeforeSourceOrModelWork(t *testing.T) {
}
})
}
if promptSource.opens.Load() != 0 {
t.Fatalf("invalid appended message opened prompt sources %d times", promptSource.opens.Load())
}
if modelCalls.Load() != 0 {
t.Fatalf("invalid appended message invoked the model %d times", modelCalls.Load())
}

138
backends.go
View File

@@ -1,107 +1,69 @@
package promptkit
import (
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
// BackendOpenRouter is the reserved ID of Promptkit's built-in OpenRouter
// backend.
// BackendOpenRouter identifies the maintained OpenRouter backend.
const BackendOpenRouter = backend.OpenRouterID
// BackendRakestrawHome is the reserved ID of Promptkit's built-in
// Rakestrawhome backend.
// BackendRakestrawHome identifies the maintained Rakestrawhome backend.
const BackendRakestrawHome = backend.RakestrawHomeID
// BackendLocal is the case-sensitive conventional ID used by [LocalBackend].
// It is not a built-in or reserved backend and must be registered with
// [WithBackend].
// BackendLocal is a conventional ID for a caller-configured local backend.
const BackendLocal = "local"
// Backend configures one engine-scoped OpenAI-compatible backend.
//
// Backend has no stable JSON representation. Use keyed literals so additions
// to this configuration value do not break source compatibility.
type Backend struct {
// ID is the stable, case-sensitive registry key. NewEngine trims it and
// requires a non-blank value. Built-in backend IDs are reserved.
ID string
// Endpoint is the OpenAI-compatible base endpoint. NewEngine trims it and
// requires an absolute HTTP or HTTPS URL with a host and without user
// information, a query string, or a fragment. Paths are allowed.
Endpoint string
// APIKeyEnv optionally names an environment lookup source for an API key.
// NewEngine trims it and requires the portable form [A-Za-z_][A-Za-z0-9_]*.
// A direct RunRequest.APIKey takes precedence. When no usable credential is
// available, the built-in client omits Authorization; injected clients own
// their own credential-resolution behavior. Store only the name, never a
// credential value.
APIKeyEnv string
// ExtraParams contains backend-wide request defaults. Values must be
// JSON-compatible, finite, acyclic, and keyed by non-empty strings. Keys
// must not be model, session_id, messages, temperature, max_tokens, top_p,
// service_tier, reasoning_effort, or response_format. An empty map supplies
// no defaults. NewEngine deeply copies the map and rejects excessively deep
// or large values for safety.
ExtraParams map[string]any
// ConcurrencyLimit is the maximum number of simultaneous model-generation
// calls allowed for this backend within one Engine. Zero leaves the backend
// unlimited. A negative value makes NewEngine fail with ErrInvalidConfig.
ConcurrencyLimit int
// QueueCapacity controls how many additional Run or RunPrepared calls may
// be admitted beyond ConcurrencyLimit. Nil uses 1024 when ConcurrencyLimit
// is positive; a pointer uses its exact value, including zero. The pointed-to
// value must be non-negative, and QueueCapacity must be nil when
// ConcurrencyLimit is zero. Their sum must fit in an int. WithBackend copies
// the value and does not retain the pointer.
// BackendConfig is a presence-aware convenience value for a keyed backend
// definition. Nil pointer fields are omitted, while non-nil pointers preserve
// explicit zero or blank values. A nil ExtraParams map is omitted; a non-nil
// empty map supplies an empty mapping. Use raw ConfigKit Mapping inputs for
// explicit null. BackendConfig has no stable JSON representation.
type BackendConfig struct {
// Endpoint is the required absolute HTTP(S) base URL after all layers merge.
Endpoint *string
// APIKeyEnv optionally names an environment variable, not a credential.
APIKeyEnv *string
// ConcurrencyLimit is zero for unlimited generation; negative is invalid.
ConcurrencyLimit *int
// QueueCapacity defaults to 1024 for positive concurrency when omitted.
// An explicit value, including zero, requires positive concurrency.
QueueCapacity *int
// ExtraParams supplies JSON-compatible provider request defaults.
ExtraParams map[string]any
}
// LocalBackend returns a caller-owned Backend for a conventional local
// OpenAI-compatible endpoint. It sets ID to BackendLocal and copies endpoint
// and concurrencyLimit into Endpoint and ConcurrencyLimit without
// normalization or validation. APIKeyEnv, ExtraParams, and QueueCapacity keep
// their zero values.
//
// LocalBackend does not read environment variables, register the value, or
// mutate engine or package state. Supply the returned value through
// [WithBackend]; [NewEngine] then applies the ordinary backend validation and
// concurrency semantics, including default queue capacity for a positive
// limit, unlimited behavior for zero, and ErrInvalidConfig for a negative
// limit.
func LocalBackend(endpoint string, concurrencyLimit int) Backend {
return Backend{
ID: BackendLocal,
Endpoint: endpoint,
ConcurrencyLimit: concurrencyLimit,
// BackendSource creates a labeled ConfigKit mapping from keyed backend values.
// It encodes fields without validating them; NewEngine validates the final
// effective configuration after all selected source layers are combined.
// Mapping captures a copy of supported input values at creation; unsupported
// values fail when the source is loaded.
func BackendSource(label string, backends map[string]BackendConfig) configkit.Source {
definitions := make(map[string]any, len(backends))
for id, config := range backends {
fields := make(map[string]any)
if config.Endpoint != nil {
fields["endpoint"] = *config.Endpoint
}
if config.APIKeyEnv != nil {
fields["api_key_env"] = *config.APIKeyEnv
}
if config.ConcurrencyLimit != nil {
fields["concurrency_limit"] = *config.ConcurrencyLimit
}
if config.QueueCapacity != nil {
fields["queue_capacity"] = *config.QueueCapacity
}
if config.ExtraParams != nil {
fields["extra_params"] = providerMapForMapping(config.ExtraParams)
}
definitions[id] = fields
}
return configkit.Mapping(label, map[string]any{"backends": definitions})
}
// WithBackend adds one Backend registration to the constructed Engine.
//
// Registrations accumulate in option order. Every normalized ID must be unique
// across consumer registrations and built-ins; a duplicate or invalid
// definition makes NewEngine fail with ErrInvalidConfig. Built-in IDs,
// including [BackendOpenRouter] and [BackendRakestrawHome], cannot be
// replaced. The immutable registration is scoped to the resulting Engine and
// cannot be enumerated, replaced, removed, or mutated after construction.
// WithBackend does not install package-global state.
func WithBackend(backend Backend) Option {
queueCapacity := 0
queueCapacitySet := backend.QueueCapacity != nil
if queueCapacitySet {
queueCapacity = *backend.QueueCapacity
}
return optionFunc(func(options *engineOptions) error {
options.internal.AddBackend(domain.Backend{
ID: backend.ID,
Endpoint: backend.Endpoint,
APIKeyEnv: backend.APIKeyEnv,
ExtraParams: backend.ExtraParams,
ConcurrencyLimit: backend.ConcurrencyLimit,
QueueCapacity: queueCapacity,
QueueCapacitySet: queueCapacitySet,
})
return nil
})
// LocalBackend supplies endpoint and concurrency pointers for a caller-chosen
// backend ID; it does not select an ID or validate either value.
func LocalBackend(endpoint string, concurrencyLimit int) BackendConfig {
return BackendConfig{Endpoint: &endpoint, ConcurrencyLimit: &concurrencyLimit}
}

17
capacity_contract_test.go
View File

@@ -8,6 +8,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/promptkit"
"gitea.maximumdirect.net/eric/promptkit/testfixture"
)
func TestEngineLimitsInjectedClientConcurrency(t *testing.T) {
@@ -150,9 +151,9 @@ func TestUnlimitedBackendsRetainInjectedClientConcurrency(t *testing.T) {
{
name: "endpoint-only profile",
configure: func(t *testing.T, client promptkit.LLMClient) *promptkit.Engine {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
promptkit.WithProfiles(promptkit.Profile{
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
testfixture.WithProfiles(testfixture.Profile{
ID: "profile", Endpoint: "http://endpoint.example/v1", Model: "model",
}),
promptkit.WithLLMClient(client),
@@ -254,15 +255,15 @@ func newBackendCapacityEngine(
if reader != nil {
promptFS = contractInputPromptFS()
}
options := []promptkit.Option{
promptkit.WithPromptFS(promptFS, "."),
promptkit.WithBackend(promptkit.Backend{
options := []any{
testfixture.WithPromptFS(promptFS, "."),
testfixture.WithBackend(testfixture.Backend{
ID: "limited",
Endpoint: "http://backend.example/v1",
ConcurrencyLimit: limit,
QueueCapacity: queueCapacity,
}),
promptkit.WithProfiles(promptkit.Profile{
testfixture.WithProfiles(testfixture.Profile{
ID: "profile", BackendID: "limited", Model: "model",
}),
promptkit.WithLLMClient(client),
@@ -270,7 +271,7 @@ func newBackendCapacityEngine(
if reader != nil {
options = append(options, promptkit.WithArtifactReader(reader))
}
engine, err := promptkit.NewEngine(promptkit.Config{}, options...)
engine, err := testfixture.NewEngine(testfixture.Config{}, options...)
if err != nil {
t.Fatalf("construct capacity engine: %v", err)
}

13
configuration_sources.go
View File

@@ -12,8 +12,10 @@ import (
)
// DirectoryInputs selects the YAML files beneath an explicitly chosen host
// directory as one strict-composition tier. The returned order is deterministic;
// it does not assign precedence among those files.
// directory as one strict-composition group. The returned order is deterministic;
// it does not assign precedence among those files. Discovery is recursive over
// .yaml and .yml files. An unavailable root fails; an empty directory yields
// no inputs. Imports are disabled unless callers enable them on the inputs.
func DirectoryInputs(directory string) ([]configkit.Input, error) {
root, err := filepath.Abs(directory)
if err != nil {
@@ -45,8 +47,11 @@ func DirectoryInputs(directory string) ([]configkit.Input, error) {
}
// FilesystemInputs selects the YAML files beneath root in a caller-owned
// filesystem as one strict-composition tier. The caller retains ownership of
// the filesystem; imports remain disabled unless callers enable them on inputs.
// filesystem as one strict-composition group. Discovery is recursive over
// .yaml and .yml files, with deterministic ordering and no precedence among
// files. The caller retains ownership of the filesystem; an unavailable root
// fails and an empty directory yields no inputs. Imports remain disabled unless
// callers enable them on inputs.
func FilesystemInputs(label string, filesystem fs.FS, root string) ([]configkit.Input, error) {
if filesystem == nil {
return nil, fmt.Errorf("configuration filesystem is nil")

174
construction_contract_test.go Normal file
View File

@@ -0,0 +1,174 @@
package promptkit_test
import (
"context"
"errors"
"os"
"path/filepath"
"testing"
"testing/fstest"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestPublicConstructionWithLocalBackendAndProgrammaticProfile(t *testing.T) {
backendID, model := "workstation", "offline-model"
prompt := configkit.Input{Source: configkit.Mapping("prompt", map[string]any{
"prompts": map[string]any{"hello": map[string]any{"1": map[string]any{
"default_profile": "offline",
"messages": []any{map[string]any{"role": "user", "content": "Hello"}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
}}},
})}
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "reply"}}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{
{Source: promptkit.BackendSource("local backend", map[string]promptkit.BackendConfig{
backendID: promptkit.LocalBackend("http://localhost:8000/v1", 2),
})},
{Source: promptkit.ProfileSource("profile", map[string]promptkit.ProfileConfig{
"offline": {Backend: &backendID, Model: &model},
})},
}}},
Main: &prompt,
}}, promptkit.WithLLMClient(client))
if err != nil {
t.Fatalf("construct engine: %v", err)
}
result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "hello"})
if err != nil {
t.Fatalf("run prompt: %v", err)
}
if result.RawOutput != "reply" || result.SelectedBackendID != backendID || result.EffectiveModelParams.Model != model {
t.Fatalf("unexpected result: %+v", result)
}
if len(client.requests) != 1 || client.requests[0].Target.Endpoint != "http://localhost:8000/v1" {
t.Fatalf("unexpected model request: %+v", client.requests)
}
}
func TestPublicConstructionCombinesEmbeddedOperatorAndProgrammaticInputs(t *testing.T) {
embedded, err := promptkit.FilesystemInputs("embedded defaults", fstest.MapFS{
"config/default.yaml": &fstest.MapFile{Data: []byte(`
profiles:
application:
endpoint: https://example.invalid/v1
model: default-model
prompts:
hello:
"1":
default_profile: application
messages:
- role: user
content: Hello
output:
format: text
validation_mode: none
`)},
}, "config")
if err != nil {
t.Fatalf("select embedded defaults: %v", err)
}
operatorDirectory := t.TempDir()
if err := os.WriteFile(filepath.Join(operatorDirectory, "override.yaml"), []byte("profiles:\n application:\n model: operator-model\n"), 0o644); err != nil {
t.Fatalf("write operator input: %v", err)
}
operator, err := promptkit.DirectoryInputs(operatorDirectory)
if err != nil {
t.Fatalf("select operator inputs: %v", err)
}
programmaticModel := "programmatic-model"
programmatic := configkit.Input{Source: promptkit.ProfileSource("programmatic settings", map[string]promptkit.ProfileConfig{
"application": {Model: &programmaticModel},
})}
for _, test := range []struct {
name string
apply bool
want string
}{
{name: "embedded", want: "default-model"},
{name: "operator", apply: true, want: "operator-model"},
{name: "programmatic", apply: true, want: "programmatic-model"},
} {
t.Run(test.name, func(t *testing.T) {
underlays := []configkit.Underlay{{Inputs: embedded}}
if test.apply {
underlays = append(underlays, configkit.Underlay{Inputs: operator})
}
if test.name == "programmatic" {
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{programmatic}})
}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: underlays}})
if err != nil {
t.Fatalf("construct engine: %v", err)
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "hello"})
if err != nil {
t.Fatalf("prepare prompt: %v", err)
}
if prepared.EffectiveModelParams.Model != test.want {
t.Fatalf("model = %q, want %q", prepared.EffectiveModelParams.Model, test.want)
}
})
}
}
func TestPublicConstructionPreservesConfigKitLoadError(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("invalid", map[string]any{"profiles": map[string]any{"broken": map[string]any{"unknown": true}}}),
}}})
var loadErr *configkit.LoadError
if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) || !errors.As(err, &loadErr) {
t.Fatalf("construction = (%#v, %v), want invalid config preserving LoadError", engine, err)
}
}
func TestPublicConstructionRejectsSuppliedInvalidMainInput(t *testing.T) {
main := configkit.Input{}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &main}})
if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("construction = (%#v, %v), want invalid supplied main input", engine, err)
}
}
func TestPublicConstructionCanOverlayMaintainedBackend(t *testing.T) {
endpoint := "http://local-openrouter.example/v1"
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.BackendSource("operator backend", map[string]promptkit.BackendConfig{
promptkit.BackendOpenRouter: {Endpoint: &endpoint},
})}}}},
}})
if err != nil {
t.Fatalf("construct engine with backend overlay: %v", err)
}
inspection, err := engine.InspectProfile(context.Background(), "mistral-small-3")
if err != nil {
t.Fatalf("inspect catalog profile: %v", err)
}
if inspection.EffectiveModelParams.Endpoint != endpoint {
t.Fatalf("effective endpoint = %q, want %q", inspection.EffectiveModelParams.Endpoint, endpoint)
}
}
func TestPreparationRejectsAmbiguousPromptVersion(t *testing.T) {
definition := func(content string) map[string]any {
return map[string]any{
"messages": []any{map[string]any{"role": "user", "content": content}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
}
}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Main: &configkit.Input{Source: configkit.Mapping("versions", map[string]any{
"prompts": map[string]any{"hello": map[string]any{
"1": definition("first"), "2": definition("second"),
}},
})},
}})
if err != nil {
t.Fatalf("construct engine: %v", err)
}
if result, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "hello"}); result != nil || !errors.Is(err, promptkit.ErrPromptLoad) {
t.Fatalf("ambiguous preparation = (%#v, %v), want ErrPromptLoad", result, err)
}
}

11
doc.go
View File

@@ -1,8 +1,8 @@
// Package promptkit provides an embeddable engine for preparing and executing
// prompt-defined LLM workflows.
//
// Applications construct an [Engine] with [NewEngine], select filesystem or
// in-memory sources and optional engine-scoped [Backend] registrations, and
// Applications construct an [Engine] with [NewEngine], select ConfigKit
// sources for prompts, profiles, and backends, and
// call [Engine.InspectPrompt], [Engine.InspectProfile], [Engine.Prepare],
// [Engine.PrepareExecution], [Engine.Run], or [Engine.RunPrepared]. Concrete
// registries, repositories, validators, and the built-in OpenAI-compatible
@@ -17,7 +17,8 @@
// independently. An injected [LLMClient] or [ArtifactReader] can therefore
// still receive concurrent calls and must be safe for that use.
//
// NewEngine copies in-memory profiles and backend definitions. Prepare,
// ConfigKit captures the selected configuration when NewEngine constructs an
// Engine. Prepare,
// PrepareExecution, and Run copy request maps, slices, pointer values, and
// JSON-compatible extra parameters before using them. InspectPrompt and
// InspectProfile return copied inspection values. Returned values and values
@@ -52,8 +53,8 @@
// used by those values.
//
// Construction, inspection, handle, and error values, including [Config],
// [Backend], [RunRequest], [ArtifactRef], [ExecutionTargetOverride], [Profile],
// [OpenAICompatibleProfileConfig], [ProfileInspection],
// [ConfigurationSources], [BackendConfig], [ProfileConfig], [RunRequest],
// [ArtifactRef], [ExecutionTargetOverride], [ProfileInspection],
// [PromptInputDefinition], [PromptInspection], [PreparedExecution],
// [CapacityError], [GenerationError], [RetryExhaustedError], and
// [ExecutionEvent], do not have stable JSON

221
docs/consumers/pkg-promptkit.md
View File

@@ -21,54 +21,62 @@ programs.
## Construct An Engine
Create an engine with
[`NewEngine`](../../engine.go). A directory-backed setup supplies a prompt
directory and may supply profile and schema directories:
Create an engine with [`NewEngine`](../../engine.go). Select each directory
explicitly and place its inputs in an ordered ConfigKit underlay. A group
strictly composes its files; later groups can overlay earlier ones:
```go
promptInputs, err := promptkit.DirectoryInputs("prompts")
if err != nil { return err }
profileInputs, err := promptkit.DirectoryInputs("profiles")
if err != nil { return err }
engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: "prompts",
ProfileDir: "profiles",
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{
{Inputs: promptInputs},
{Inputs: profileInputs},
}},
SchemaDir: "schemas",
})
```
Options support single-file or `fs.FS` sources, in-memory profiles,
engine-scoped backends, and injected artifact or model clients. Consult the
[constructor and option GoDoc](../../engine.go) for composition, precedence,
validation, and default transport behavior. Source discovery, format
validation, and profile precedence are defined by the
[framework format reference](../formats.md).
For a single highest-priority document, set `Sources.Main` to a
`*configkit.Input`; omit it for ordinary layered construction. A single file
can use `configkit.File` directly. `ProfileSource`, `BackendSource`, and raw
`configkit.Mapping` supply programmatic definitions. Schema and runtime
collaborator options remain available. Construction validates the final
effective configuration and captures referenced prompt content; construct a
new engine to observe configuration edits. See the [constructor GoDoc](../../engine.go)
for the exact public contract and the [framework format reference](../formats.md)
for definition fields.
## Supply Embedded Application Defaults
Use `WithFallbackProfileFS` when an application packages profile definitions
that should apply unless an operator provides an ordinary configured profile
with the same ID. For example, an application can embed its defaults while
continuing to use `ProfileDir` for operator overrides:
Use `FilesystemInputs` for embedded defaults and `DirectoryInputs` for an
explicitly chosen operator directory. A later programmatic group can override
both. No directory name has built-in precedence:
```go
//go:embed profiles/*.yaml
var applicationProfiles embed.FS
engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: "prompts",
ProfileDir: operatorProfileDir,
defaults, err := promptkit.FilesystemInputs("embedded defaults", applicationFS, "config")
if err != nil { return err }
operator, err := promptkit.DirectoryInputs(operatorDirectory)
if err != nil { return err }
profile := promptkit.ProfileSource("application choices", map[string]promptkit.ProfileConfig{
"summary": {Model: &selectedModel},
})
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{
{Inputs: defaults},
{Inputs: operator},
{Inputs: []configkit.Input{{Source: profile}}},
},
promptkit.WithFallbackProfileFS(applicationProfiles, "profiles"),
)
}})
```
Keep application-owned profile IDs and definitions in the embedded source.
Use the ordinary configured profile source for operator overrides. Leave
`operatorProfileDir` empty when the operator did not configure an override
directory; a non-empty path names an authoritative higher-precedence source,
so an unavailable or unreadable directory is an error rather than a reason to
fall back. The
[framework format reference](../formats.md#source-and-profile-precedence)
owns the exact profile format and lookup order; the
[`WithFallbackProfileFS` GoDoc](../../engine.go) owns its option contract and
validation rules.
Only add the operator group when the application selects that directory; an
explicitly selected missing or unreadable root is an error. Each helper finds
`.yaml` and `.yml` files recursively and disables imports by default. The
caller may enable imports on individual inputs. A nil `Sources.Main` is an
empty main source, not another directory lookup.
## Inspect A Prompt Before Preparation
@@ -268,8 +276,8 @@ result, err := engine.Run(ctx, request)
These messages are already rendered: Promptkit does not template or resolve
files in them, and they can contain sensitive model output or application
feedback. Promptkit remains stateless; every `Run` call re-resolves its current
sources and the application owns any semantic retry budget. When a
feedback. Every `Run` call uses the engine's captured configuration; the
application owns any semantic retry budget. When a
pre-execution equality check is required, use `PrepareExecution` and compare
its opaque rendered-prompt hash before invoking `RunPrepared`.
@@ -282,12 +290,12 @@ replace execution settings or the complete output contract.
The [public value GoDoc](../../types.go) defines nil, empty, zero, replacement,
copy, and credential behavior. The
[framework format reference](../formats.md) defines how those request values
interact with prompt definitions, file-backed and application fallback
profiles, built-ins, schemas, and framework defaults.
interact with prompt definitions, layered profiles, built-ins, schemas, and
framework defaults.
For programmatic profiles,
[`OpenAICompatibleProfile`](../../profiles.go) converts ordinary
OpenAI-compatible settings into a value accepted by `WithProfiles`.
For programmatic profiles, use [`ProfileSource`](../../profiles.go) with keyed
`ProfileConfig` values. Nil fields are omitted; pointers preserve explicit
zero, false, and blank values. Raw `configkit.Mapping` expresses explicit null.
### Choose A Profile Ownership Model
@@ -319,18 +327,19 @@ the application ID while inheriting the built-in target. The child can override
only the setting it owns:
```go
promptkit.WithProfiles(promptkit.Profile{
ID: "weather-light",
BaseProfileID: "promptkit-openrouter-standard",
ReasoningEffort: "high",
profile := promptkit.ProfileSource("application profiles", map[string]promptkit.ProfileConfig{
"weather-light": {
Extends: &standardID,
ReasoningEffort: &high,
},
})
```
Select `weather-light` in a prompt or `RunRequest.ProfileID`; it remains the
reported selected profile. See the [profile inheritance format
reference](../formats.md#profile-inheritance) and the
[`Profile` GoDoc](../../types.go) for exact lookup, merging, and validation
behavior. If the base is an external profile, apply the volatility guidance
Place `profile` in a ConfigKit underlay, with `standardID` set to
`"promptkit-openrouter-standard"` and `high` to `"high"`. Select
`weather-light` in a prompt or `RunRequest.ProfileID`; it remains the
reported selected profile. Definition IDs and inheritance references are exact
keys. If the base is an external profile, apply the volatility guidance
above when deciding whether to use an application-owned complete definition
instead.
@@ -355,9 +364,8 @@ if prepared.SelectedBackendID != promptkit.BackendRakestrawHome {
}
```
Do not register `rakestrawhome` manually. When adopting this built-in, remove
an existing `WithBackend` registration with that exact ID; retaining it causes
the intentional duplicate-ID configuration error. Direct request credentials
The maintained catalog supplies this backend, and callers may overlay its
effective configuration through a later ConfigKit source. Direct request credentials
and runtime endpoint overrides remain supported under their ordinary GoDoc and
format contracts.
@@ -424,19 +432,15 @@ Choose the smallest configuration that fits how the endpoint will be reused.
#### Use An Endpoint-Only Profile
Put the endpoint directly on an in-memory profile when only that profile needs
Put the endpoint directly on a programmatic profile when only that profile needs
it and shared backend identity or capacity policy is unnecessary:
```go
engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: "prompts",
},
promptkit.WithProfiles(promptkit.Profile{
ID: "local-summary",
Endpoint: "http://localhost:8000/v1",
Model: "example-model",
}),
)
endpoint, model := "http://localhost:8000/v1", "example-model"
profile := promptkit.ProfileSource("local profile", map[string]promptkit.ProfileConfig{
"local-summary": {Endpoint: &endpoint, Model: &model},
})
// Add configkit.Input{Source: profile} to a caller underlay.
```
Endpoint-only profiles have an empty backend ID and remain unrestricted by
@@ -448,59 +452,44 @@ Use `LocalBackend` when profiles should share the conventional `local`
identity, endpoint, and concurrency limit:
```go
engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: "prompts",
},
promptkit.WithBackend(
promptkit.LocalBackend("http://localhost:8000/v1", 2),
),
promptkit.WithProfiles(promptkit.Profile{
ID: "local-summary",
BackendID: promptkit.BackendLocal,
Model: "example-model",
}),
)
backendID, model := promptkit.BackendLocal, "example-model"
backend := promptkit.BackendSource("local backend", map[string]promptkit.BackendConfig{
backendID: promptkit.LocalBackend("http://localhost:8000/v1", 2),
})
profile := promptkit.ProfileSource("local profile", map[string]promptkit.ProfileConfig{
"local-summary": {Backend: &backendID, Model: &model},
})
// Add both sources as inputs in a caller underlay.
```
The helper is explicit: it does not pre-register a backend or read environment
variables. Supplying a positive limit leaves queue capacity omitted, so normal
backend registration selects the existing default waiting capacity of 1024.
The returned value still enters the engine through `WithBackend`.
The helper does not choose an ID or read environment variables. A positive
concurrency limit with omitted queue capacity selects the default waiting
capacity of 1024.
#### Configure A Complete Backend
Use a keyed `Backend` value for authentication, extra request parameters, an
Use a keyed `BackendConfig` value for authentication, extra request parameters, an
explicit queue capacity, a custom ID, or multiple local endpoints:
```go
noWaiting := 0
engine, err := promptkit.NewEngine(promptkit.Config{
PromptDir: "prompts",
},
promptkit.WithBackend(promptkit.Backend{
ID: "local-gpu",
Endpoint: "http://gpu-host:8000/v1",
APIKeyEnv: "LOCAL_GPU_API_KEY",
endpoint, keyName, limit, noWaiting := "http://gpu-host:8000/v1", "LOCAL_GPU_API_KEY", 2, 0
backend := promptkit.BackendSource("GPU backend", map[string]promptkit.BackendConfig{
"local-gpu": {
Endpoint: &endpoint, APIKeyEnv: &keyName,
ConcurrencyLimit: &limit, QueueCapacity: &noWaiting,
ExtraParams: map[string]any{"provider_option": "enabled"},
ConcurrencyLimit: 2,
QueueCapacity: &noWaiting,
}),
promptkit.WithProfiles(promptkit.Profile{
ID: "gpu-summary",
BackendID: "local-gpu",
Model: "example-model",
}),
)
},
})
```
Use distinct custom IDs when registering multiple local endpoints.
Registrations belong to one engine and custom IDs cannot replace built-ins.
The [`Backend`, `LocalBackend`, and `WithBackend` GoDoc](../../backends.go)
defines exact construction, validation, copying, uniqueness, concurrency, and
request-default behavior.
Place `backend` in a caller underlay and select `"local-gpu"` from a
`ProfileConfig.Backend` pointer or a keyed file profile. All backend
contributions use the same ConfigKit layering rules, including overlays of
maintained definitions. The [`BackendConfig` and `LocalBackend` GoDoc](../../backends.go)
defines exact omission and construction behavior.
Both file-backed and in-memory profiles select a registration through
`backend` or `Profile.BackendID`. Profile and request endpoint overrides retain
Both file-backed and programmatic profiles select a backend through
`backend` or `ProfileConfig.Backend`. Profile and request endpoint overrides retain
that routing and capacity identity. `PreparedRun.SelectedBackendID`,
`RunResult.SelectedBackendID`, and the effective `ExecutionTarget.BackendID`
expose it to consumers and injected model clients. Endpoint-only profiles
@@ -508,7 +497,7 @@ remain supported and expose an empty backend ID.
### Limit Backend Concurrency
Set `Backend.ConcurrencyLimit` when a backend needs protection from too many
Set `BackendConfig.ConcurrencyLimit` when a backend needs protection from too many
simultaneous model calls. Leaving `QueueCapacity` nil, as in the local-backend
example above, selects the default waiting capacity of 1024.
@@ -517,24 +506,23 @@ zero:
```go
noWaiting := 0
backend := promptkit.Backend{
ID: "local-gpu",
Endpoint: "http://gpu-host:8000/v1",
ConcurrencyLimit: 2,
backend := promptkit.BackendConfig{
Endpoint: &endpoint,
ConcurrencyLimit: &limit,
QueueCapacity: &noWaiting,
}
```
The pointer distinguishes an explicit zero from omission. Keep using keyed
`Backend` literals so additive configuration fields remain source-compatible.
The pointer distinguishes an explicit zero from omission. Use keyed
`BackendConfig` literals inside `BackendSource` for programmatic definitions.
Capacity belongs to one engine and the selected backend ID; endpoint-only
profiles and custom backends without a configured limit remain unrestricted.
Exact validation, defaulting, ownership, and concurrency semantics belong to
the [`Backend` GoDoc](../../backends.go).
the [`BackendConfig` GoDoc](../../backends.go).
## Credentials
File-backed profiles name an environment variable; in-memory profiles can
File-backed and programmatic profiles can name an environment variable or
require a direct request key. Direct keys are request-scoped and are excluded
from supported JSON values and the package's `String` and `GoString`
summaries. The exact precedence and redaction guarantees belong to
@@ -568,12 +556,13 @@ the application policy that motivated the injection.
Use `errors.Is` with the
[public error sentinels and operation GoDoc](../../engine.go). The declarations
distinguish invalid construction, invalid requests, absent sources,
source-loading failures, collaborator failures, and operational validation
distinguish invalid construction, invalid requests, absent selections,
collaborator failures, and operational validation
failures. Specific request conditions may also match the broader
`ErrInvalidRequest`, and injected collaborator identities are preserved where
documented. Invalid or duplicate backend registrations match
`ErrInvalidConfig`; selecting an unknown backend matches `ErrProfileLoad`.
documented. Invalid effective definitions, including unknown backend
references, match `ErrInvalidConfig` during construction. ConfigKit load
errors remain available through `errors.As`.
When a limited backend has admitted all active and waiting calls, handle
`ErrCapacityExceeded` separately from request errors and provider failures:

2
docs/roadmap/implementation.md
View File

@@ -465,6 +465,8 @@ pass against released modules with `GOWORK=off` and no replacements.
## Stage 7 — Cut Over Engine Assembly And Public Construction
**Completed:** 2026-09-30.
Scope: `engine.go`, configuration facade/helpers, `internal/engine`, facade
adapters, and migration of their direct test setup.

205
engine.go
View File

@@ -8,6 +8,7 @@ import (
"net/http"
"time"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
internalengine "gitea.maximumdirect.net/eric/promptkit/internal/engine"
"gitea.maximumdirect.net/eric/promptkit/internal/observation"
@@ -15,7 +16,7 @@ import (
)
// ErrInvalidConfig identifies invalid engine construction, including missing
// required configuration, invalid options or backend registrations, and a nil
// effective configuration, invalid options, and a nil
// Engine receiver.
var ErrInvalidConfig = errors.New("invalid engine configuration")
@@ -34,13 +35,13 @@ var (
// nor the selected prompt's default profile is present. Such an error also
// matches ErrInvalidRequest.
ErrProfileRequired = errors.New("profile selection is required")
// ErrPromptLoad identifies a failure to read, decode, validate, select, or
// hash a prompt definition, except for the not-found case represented by
// ErrPromptNotFound.
// ErrPromptLoad identifies a prompt selection failure other than the
// not-found case represented by ErrPromptNotFound. Invalid definitions
// instead fail construction with ErrInvalidConfig.
ErrPromptLoad = errors.New("failed to load prompt definition")
// ErrProfileLoad identifies a failure to read, decode, validate, or select
// an execution profile or resolve its backend, except for the profile
// not-found case represented by ErrProfileNotFound.
// ErrProfileLoad identifies a profile selection or resolution failure other
// than the not-found case represented by ErrProfileNotFound. Invalid
// definitions instead fail construction with ErrInvalidConfig.
ErrProfileLoad = errors.New("failed to load execution profile")
// ErrAPIKeyEnvMissing identifies an explicitly required APIKeyEnv whose
// environment variable is unset or empty after direct RunRequest.APIKey
@@ -94,18 +95,21 @@ type Engine struct {
internal *internalengine.Engine
}
// Config selects the directory-backed sources and built-in model-client
// transport used by [NewEngine]. Config has no stable JSON representation.
// ConfigurationSources selects ordered ConfigKit underlays and an optional
// highest-priority main input. Each underlay's inputs strictly compose within
// their group; later underlays overlay earlier ones. A nil Main uses
// ConfigKit's empty source. A non-nil Main is passed through unchanged and
// may fail construction if invalid.
type ConfigurationSources struct {
Underlays []configkit.Underlay
Main *configkit.Input
}
// Config selects configuration sources and built-in model-client transport
// used by [NewEngine]. Config has no stable JSON representation.
type Config struct {
// PromptDir is the directory searched recursively for prompt definitions.
// It is required unless a WithPromptFS or WithPromptFile option supplies the
// prompt source.
PromptDir string
// ProfileDir is an optional ordinary configured source whose profiles take
// precedence over application fallback and maintained catalog profiles. An
// empty value selects the lower-precedence sources unless a profile-source
// option supplies the ordinary source.
ProfileDir string
// Sources combines maintained definitions with caller-selected inputs.
Sources ConfigurationSources
// SchemaDir is the root for JSON Schema files. An empty value uses the
// current directory. WithSchemaFS or WithSchemaFile replaces this source.
SchemaDir string
@@ -128,12 +132,9 @@ type Config struct {
// Option customizes engine construction.
//
// NewEngine applies options in argument order and ignores nil options. Within
// each prompt-source, ordinary-profile-source, fallback-profile-source,
// in-memory-profile, schema-source, model-client, and artifact-reader
// category, the last non-nil valid option replaces earlier options in that
// category. WithBackend is the additive exception: unique registrations
// accumulate, and a repeated backend ID is an error rather than a replacement.
// An invalid option fails construction even if a later option would replace it.
// each schema-source, model-client, and artifact-reader category, the last
// non-nil valid option replaces earlier options. An invalid option fails
// construction even if a later option would replace it.
type Option interface {
apply(*engineOptions) error
}
@@ -181,102 +182,6 @@ func WithArtifactReader(reader ArtifactReader) Option {
})
}
// WithPromptFS loads prompt definitions from fsys under root.
//
// The source uses the same strict prompt YAML rules as configured prompt
// directories, and prompt content_file paths resolve within this source.
// fsys must be non-nil and root must be non-empty; otherwise NewEngine fails
// with ErrInvalidConfig. This option replaces Config.PromptDir and earlier
// prompt-source options.
func WithPromptFS(fsys fs.FS, root string) Option {
return optionFunc(func(options *engineOptions) error {
return options.internal.SetPromptFS(fsys, root)
})
}
// WithPromptFile loads prompt definitions from the single prompt file at path.
//
// Relative prompt content_file paths resolve from the file's directory. path
// must name an existing non-directory file when NewEngine applies the option.
// This option replaces Config.PromptDir and earlier prompt-source options.
func WithPromptFile(path string) Option {
return optionFunc(func(options *engineOptions) error {
return options.internal.SetPromptFile(path)
})
}
// WithProfileFS loads execution profiles from fsys under root.
//
// Profiles from this ordinary configured source take precedence over
// application fallback and built-in profiles. Profile YAML must use api_key_env
// for environment-based credentials; raw API keys are rejected. fsys must be
// non-nil and root must be non-empty; otherwise NewEngine fails with
// ErrInvalidConfig. This option replaces Config.ProfileDir and earlier file or
// FS profile-source options, but remains below WithProfiles in precedence.
func WithProfileFS(fsys fs.FS, root string) Option {
return optionFunc(func(options *engineOptions) error {
return options.internal.SetProfileFS(fsys, root)
})
}
// WithProfileFile loads execution profiles from the single profile file at path.
//
// The profile takes precedence over application fallback and built-in profiles.
// Profile YAML must use api_key_env for environment-based credentials; raw API
// keys are rejected. path must name an existing non-directory file when
// NewEngine applies the option. This option replaces Config.ProfileDir and
// earlier file or FS profile-source options, but remains below WithProfiles in
// precedence.
func WithProfileFile(path string) Option {
return optionFunc(func(options *engineOptions) error {
return options.internal.SetProfileFile(path)
})
}
// WithFallbackProfileFS supplies application-owned fallback profile
// definitions from fsys under root.
//
// Profile lookup checks, in order, profiles supplied by WithProfiles; the
// ordinary configured source selected by WithProfileFile, WithProfileFS, or
// Config.ProfileDir; this fallback source; and Promptkit's maintained catalog
// profiles. Each source supplies a complete profile definition; profile fields
// are not merged between sources. Only an absent profile ID proceeds to the
// next source. A matching read, parse, duplicate, validation, or credential
// format failure stops resolution.
//
// Files use the ordinary strict profile YAML and api_key_env credential rules.
// Loading and validation are lazy: NewEngine validates this option's arguments
// but does not read profile files. fsys must be non-nil and root must be
// nonblank; otherwise NewEngine returns an error matching ErrInvalidConfig.
// Repeating this option replaces the earlier valid fallback source.
//
// This option controls profile-definition lookup, not provider or generation
// failover.
func WithFallbackProfileFS(fsys fs.FS, root string) Option {
return optionFunc(func(options *engineOptions) error {
return options.internal.SetFallbackProfileFS(fsys, root)
})
}
// WithProfiles configures in-memory profiles that take precedence over
// ordinary configured, application fallback, and built-in profiles.
//
// NewEngine locally validates and copies every profile. IDs must be unique
// within one call. An invalid local definition, duplicate ID, or unsupported
// ExtraParams value makes construction fail with ErrInvalidConfig. A derived
// profile's base reference and resolved target completeness are checked when it
// is selected or inspected. Repeating WithProfiles replaces the complete
// earlier in-memory set rather than merging it.
func WithProfiles(profiles ...Profile) Option {
return optionFunc(func(options *engineOptions) error {
definitions := make([]domain.ExecutionProfile, len(profiles))
for index, publicProfile := range profiles {
definitions[index] = toDomainProfile(publicProfile)
}
return options.internal.SetMemoryProfiles(definitions)
})
}
// WithSchemaFS loads JSON Schema documents from fsys under root.
//
// Prompt schema_path values resolve within this source when schema validation
@@ -302,14 +207,15 @@ func WithSchemaFile(path string) Option {
// NewEngine constructs an Engine from configuration and options.
//
// Options are applied in order according to [Option]. PromptDir is required
// unless a prompt-source option is present. Construction validates option
// arguments, in-memory profiles, and backend registrations but defers reading
// and validating prompt, file-backed profile, and schema contents until Prepare
// or Run needs them.
// ConfigKit composes maintained catalog inputs, PromptKit-owned definitions,
// caller underlays, and the optional main input in that order. Construction
// validates the final effective backend, profile, and prompt definitions and
// captures referenced prompt content. A missing prompt collection is allowed.
// Schema documents and request artifacts remain runtime resources.
//
// NewEngine returns an error matching ErrInvalidConfig for invalid
// configuration, options, or backend-capacity policies. Each constructed
// configuration or options, preserving a ConfigKit LoadError when loading
// fails. Each constructed
// Engine has independent backend-capacity pools. Construction does not perform
// model requests or require credentials. A negative Config.TransientRetries
// is invalid.
@@ -320,16 +226,16 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
continue
}
if err := opt.apply(&options); err != nil {
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
return nil, fmt.Errorf("%w: %w", ErrInvalidConfig, err)
}
}
assembled, err := internalengine.New(
internalengine.Config{PromptDir: cfg.PromptDir, ProfileDir: cfg.ProfileDir, SchemaDir: cfg.SchemaDir, Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient, TransientRetries: copyIntPtr(cfg.TransientRetries)},
internalengine.Config{Sources: internalengine.ConfigurationSources{Underlays: cfg.Sources.Underlays, Main: cfg.Sources.Main}, SchemaDir: cfg.SchemaDir, Timeout: cfg.Timeout, HTTPClient: cfg.HTTPClient, TransientRetries: copyIntPtr(cfg.TransientRetries)},
options.internal,
)
if err != nil {
return nil, fmt.Errorf("%w: %v", ErrInvalidConfig, err)
return nil, fmt.Errorf("%w: %w", ErrInvalidConfig, err)
}
return &Engine{internal: assembled}, nil
}
@@ -339,9 +245,9 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
//
// InspectPrompt requires a nonblank promptID. It passes nonblank promptID and
// promptVersion values unchanged to the engine's ordinary, case-sensitive
// prompt selection. An empty version succeeds only when that source has one
// selected ID; a nonempty version selects one exact ID/version pair. The
// configured prompt source is used without merging, fallback, or enumeration.
// prompt selection. An empty version succeeds only when that ID has one
// version; a nonempty version selects one exact ID/version pair from the
// captured configuration snapshot.
//
// A successful result proves that the selected definition and any referenced
// message content files were structurally loaded. Inputs are returned in
@@ -349,22 +255,22 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
// resolved. OutputContract is the normalized declared contract, with a JSON
// Schema path when declared but without loading or compiling that schema.
// PromptHash is the same opaque equality value as PreparedRun.PromptHash for
// the selected definition and observed source state; its spelling, length,
// the selected captured definition; its spelling, length,
// encoding, algorithm, and security properties are not contracts.
//
// This method does not return prompt bodies, templates, source paths, schemas,
// rendered messages, or execution settings. It does not resolve a profile or
// credential, read artifacts or schemas, render, validate, admit capacity,
// contact a provider, or generate model output. The returned PromptInspection
// and its input slice are caller-owned. Filesystem-backed inspection is a
// point-in-time lookup and does not freeze a definition for later execution.
// and its input slice are caller-owned. Inspection does not reload configuration
// or content files; construct another Engine to observe source changes.
//
// A nil Engine returns an error matching ErrInvalidConfig. A blank prompt ID
// matches ErrInvalidRequest. An absent exact ID or version matches
// ErrPromptNotFound and not ErrPromptLoad. Malformed, unreadable, duplicate,
// ambiguous, referenced-content, or hashing failures match ErrPromptLoad.
// Cancellation during lookup matches ErrPromptLoad while preserving the
// context error. InspectPrompt returns no partial result on error.
// ErrPromptNotFound and not ErrPromptLoad. An ambiguous omitted version matches
// ErrPromptLoad. Invalid definitions and resource failures reject construction
// instead. Cancellation during lookup preserves the context error. InspectPrompt
// returns no partial result on error.
func (e *Engine) InspectPrompt(
ctx context.Context,
promptID string,
@@ -384,10 +290,9 @@ func (e *Engine) InspectPrompt(
// InspectProfile resolves one explicit profile without selecting a prompt or
// starting execution work.
//
// InspectProfile trims surrounding whitespace from profileID and looks up the
// resulting nonblank ID exactly and case-sensitively through the engine's
// in-memory, ordinary configured-source, application fallback, and built-in
// profile precedence. It applies the framework timeout baseline, selected
// InspectProfile looks up a nonblank profileID exactly and case-sensitively
// in the captured effective configuration. It applies the framework timeout
// baseline, selected
// backend, and then selected profile to EffectiveModelParams without a request
// override. BackendID is empty for an endpoint-only profile.
//
@@ -398,16 +303,15 @@ func (e *Engine) InspectPrompt(
// absent or blank named environment variable is not an error.
//
// The returned ProfileInspection and all nested mutable values are
// caller-owned. Filesystem-backed inspection is a point-in-time lookup and
// does not freeze the profile for a later execution. This method does not load
// caller-owned. Inspection does not reload configuration; construct another
// Engine to observe source changes. This method does not load
// a prompt, render, read artifacts or schemas, admit backend capacity, contact
// a provider, or generate model output.
//
// A nil Engine returns an error matching ErrInvalidConfig. A blank profile ID
// matches ErrInvalidRequest. An absent exact ID matches ErrProfileNotFound and
// not ErrProfileLoad. Malformed or unreadable profile data, an unknown backend,
// or an invalid resolved target matches ErrProfileLoad. Cancellation during
// profile loading matches ErrProfileLoad while preserving the context error.
// not ErrProfileLoad. Invalid definitions or unknown backends reject
// construction instead. Cancellation preserves the context error.
// InspectProfile returns no partial result on error.
func (e *Engine) InspectProfile(ctx context.Context, profileID string) (*ProfileInspection, error) {
if e == nil || e.internal == nil {
@@ -495,8 +399,9 @@ func (e *Engine) PrepareExecution(ctx context.Context, req RunRequest) (*Prepare
}
// Run prepares a request, invokes the configured LLMClient, and validates the
// generated output. Each call resolves current sources and composes a fresh,
// stateless effective prompt with any validated RunRequest.AppendedMessages.
// generated output. Each call uses the Engine's captured configuration and
// composes a fresh effective prompt with any validated
// RunRequest.AppendedMessages.
//
// A content-validation failure is a successful run whose
// RunResult.Validation has Status ValidationFailed. When its output contract

517
engine_test.go
View File

File diff suppressed because it is too large Load Diff

27
examples/go-library/observe/main.go
View File

@@ -10,6 +10,7 @@ import (
"os"
"strings"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit"
)
@@ -78,14 +79,24 @@ type summary struct {
func run() (summary, error) {
transport := &scriptedTransport{}
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: transport}},
promptkit.WithPromptFS(assets, "prompts"),
promptkit.WithSchemaFS(assets, "schemas"),
promptkit.WithBackend(promptkit.Backend{ID: "offline-backend", Endpoint: "https://example.invalid/v1", ConcurrencyLimit: 1}),
promptkit.WithProfiles(
promptkit.Profile{ID: "proposal-profile", BackendID: "offline-backend", Model: "proposal-model"},
promptkit.Profile{ID: "assessment-profile", BackendID: "offline-backend", Model: "assessment-model"},
))
inputs, err := promptkit.FilesystemInputs("example prompts", assets, "prompts")
if err != nil {
return summary{}, err
}
backendID, proposalModel, assessmentModel := "offline-backend", "proposal-model", "assessment-model"
inputs = append(inputs,
configkit.Input{Source: promptkit.BackendSource("example backend", map[string]promptkit.BackendConfig{
backendID: promptkit.LocalBackend("https://example.invalid/v1", 1),
})},
configkit.Input{Source: promptkit.ProfileSource("example profiles", map[string]promptkit.ProfileConfig{
"proposal-profile": {Backend: &backendID, Model: &proposalModel},
"assessment-profile": {Backend: &backendID, Model: &assessmentModel},
})},
)
engine, err := promptkit.NewEngine(promptkit.Config{
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}},
HTTPClient: &http.Client{Transport: transport},
}, promptkit.WithSchemaFS(assets, "schemas"))
if err != nil {
return summary{}, err
}

5
examples/go-library/observe/prompts/assessment.yaml
View File

@@ -1,5 +1,6 @@
id: example.assessment
version: "1.0.0"
prompts:
example.assessment:
"1.0.0":
default_profile: assessment-profile
description: Assess a structured proposal independently of schema validity.
inputs:

5
examples/go-library/observe/prompts/proposal.yaml
View File

@@ -1,5 +1,6 @@
id: example.proposal
version: "1.0.0"
prompts:
example.proposal:
"1.0.0":
default_profile: proposal-profile
description: Propose a bounded improvement from a synthetic idea.
inputs:

17
examples/go-library/prepare/main.go
View File

@@ -6,6 +6,7 @@ import (
"fmt"
"os"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit"
)
@@ -18,15 +19,13 @@ type summary struct {
}
func main() {
engine, err := promptkit.NewEngine(
promptkit.Config{},
promptkit.WithPromptFile("examples/go-library/prepare/prompt.yaml"),
promptkit.WithProfiles(promptkit.Profile{
ID: "offline-example",
Endpoint: "https://example.invalid/v1",
Model: "offline-model",
}),
)
endpoint, model := "https://example.invalid/v1", "offline-model"
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("example profile", map[string]promptkit.ProfileConfig{
"offline-example": {Endpoint: &endpoint, Model: &model},
})}}}},
Main: &configkit.Input{Source: configkit.File("examples/go-library/prepare/prompt.yaml")},
}})
if err != nil {
exit(err)
}

5
examples/go-library/prepare/prompt.yaml
View File

@@ -1,5 +1,6 @@
id: example.prepare
version: "1.0.0"
prompts:
example.prepare:
"1.0.0":
default_profile: offline-example
description: Prepare a prompt without contacting a model provider.
inputs:

18
examples/go-library/run/main.go
View File

@@ -6,6 +6,7 @@ import (
"fmt"
"os"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit"
)
@@ -38,16 +39,13 @@ type summary struct {
}
func main() {
engine, err := promptkit.NewEngine(
promptkit.Config{},
promptkit.WithPromptFile("examples/go-library/run/prompt.yaml"),
promptkit.WithProfiles(promptkit.Profile{
ID: "offline-example",
Endpoint: "https://example.invalid/v1",
Model: "offline-model",
}),
promptkit.WithLLMClient(deterministicClient{}),
)
endpoint, model := "https://example.invalid/v1", "offline-model"
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("example profile", map[string]promptkit.ProfileConfig{
"offline-example": {Endpoint: &endpoint, Model: &model},
})}}}},
Main: &configkit.Input{Source: configkit.File("examples/go-library/run/prompt.yaml")},
}}, promptkit.WithLLMClient(deterministicClient{}))
if err != nil {
exit(err)
}

5
examples/go-library/run/prompt.yaml
View File

@@ -1,5 +1,6 @@
id: example.run
version: "1.0.0"
prompts:
example.run:
"1.0.0":
default_profile: offline-example
description: Run a prompt with a deterministic injected model client.
inputs:

9
generation_error_contract_test.go
View File

@@ -10,6 +10,7 @@ import (
"testing"
"gitea.maximumdirect.net/eric/promptkit"
"gitea.maximumdirect.net/eric/promptkit/testfixture"
)
func TestBuiltInGenerationError(t *testing.T) {
@@ -73,7 +74,7 @@ func TestBuiltInRepairGenerationError(t *testing.T) {
body := `{"error":{"code":"` + codeMarker + `","type":"` + typeMarker + `","message":"` + messageMarker + `"}}`
return &http.Response{StatusCode: http.StatusUnprocessableEntity, ContentLength: int64(len(body)), Body: io.NopCloser(strings.NewReader(body))}, nil
})}
engine, err := promptkit.NewEngine(config)
engine, err := testfixture.NewEngine(config)
if err != nil {
t.Fatalf("NewEngine: %v", err)
}
@@ -100,7 +101,7 @@ func TestBuiltInGenerationFailureCategories(t *testing.T) {
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
return nil, transportCause
})}
engine, err := promptkit.NewEngine(config)
engine, err := testfixture.NewEngine(config)
if err != nil {
t.Fatal(err)
}
@@ -143,7 +144,7 @@ func TestBuiltInGenerationPreservesContextIdentity(t *testing.T) {
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
return nil, tc.cause
})}
engine, err := promptkit.NewEngine(config)
engine, err := testfixture.NewEngine(config)
if err != nil {
t.Fatal(err)
}
@@ -217,7 +218,7 @@ func newBuiltInGenerationErrorEngine(t *testing.T, statusCode int, body string)
Body: io.NopCloser(strings.NewReader(body)),
}, nil
})}
engine, err := promptkit.NewEngine(config)
engine, err := testfixture.NewEngine(config)
if err != nil {
t.Fatalf("NewEngine: %v", err)
}

23
internal/configuration/prompts.go
View File

@@ -1,6 +1,7 @@
package configuration
import (
"context"
"errors"
"fmt"
"io"
@@ -310,3 +311,25 @@ func (state *loaded) prompt(id, version string) (*domain.PromptDefinition, domai
inspection.Inputs = append([]domain.PromptInput(nil), inspection.Inputs...)
return &definition, inspection, nil
}
// GetPromptSnapshot returns an independently owned captured definition and its
// construction-time hash. Lookup retains literal ID and version keys.
func (state *loaded) GetPromptSnapshot(ctx context.Context, id, version string) (*domain.PromptDefinition, string, error) {
if err := ctx.Err(); err != nil {
return nil, "", err
}
definition, inspection, err := state.prompt(id, version)
if errors.Is(err, errPromptNotFound) {
return nil, "", promptdef.ErrPromptDefinitionNotFound
}
if err != nil {
return nil, "", err
}
return definition, inspection.PromptHash, nil
}
// GetPromptDefinition implements the runner's prompt lookup boundary.
func (state *loaded) GetPromptDefinition(ctx context.Context, id, version string) (*domain.PromptDefinition, error) {
definition, _, err := state.GetPromptSnapshot(ctx, id, version)
return definition, err
}

11
internal/configuration/targets.go
View File

@@ -177,3 +177,14 @@ func (state *loaded) GetBackend(id string) (domain.Backend, error) {
}
return state.registry.GetBackend(id)
}
// CapacityPolicies returns the selected backend admission policies.
func (state *loaded) CapacityPolicies() map[string]domain.BackendCapacityPolicy {
if state == nil {
return nil
}
return state.registry.CapacityPolicies()
}
// ExactIDs indicates that ConfigKit definition keys are literal identities.
func (state *loaded) ExactIDs() bool { return true }

144
internal/engine/engine.go
View File

@@ -11,47 +11,40 @@ import (
"strings"
"time"
openrouter "gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2"
rakestrawhome "gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit/internal/artifact"
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/capacity"
"gitea.maximumdirect.net/eric/promptkit/internal/catalog"
"gitea.maximumdirect.net/eric/promptkit/internal/configuration"
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/generation"
"gitea.maximumdirect.net/eric/promptkit/internal/legacycatalog"
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
"gitea.maximumdirect.net/eric/promptkit/internal/prompt"
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
"gitea.maximumdirect.net/eric/promptkit/internal/usecase"
"gitea.maximumdirect.net/eric/promptkit/internal/validate"
)
// Config carries construction settings selected by the public facade.
type Config struct {
PromptDir string
ProfileDir string
Sources ConfigurationSources
SchemaDir string
Timeout time.Duration
HTTPClient *http.Client
TransientRetries *int
}
// ConfigurationSources identifies caller-selected ConfigKit layers.
type ConfigurationSources struct {
Underlays []configkit.Underlay
Main *configkit.Input
}
// Options collects construction choices from the public facade.
type Options struct {
llmClient llm.Client
artifactReader artifact.Reader
promptDefinitions promptdef.Repository
profiles profile.Repository
fallbackProfiles profile.Repository
memoryProfiles profile.Repository
backends []domain.Backend
validator validate.Validator
promptSource bool
profileSource bool
fallbackProfileSource bool
memorySource bool
validatorSource bool
artifactSource bool
}
@@ -65,72 +58,6 @@ func (o *Options) SetArtifactReader(reader artifact.Reader) {
o.artifactSource = true
}
// SetPromptFS selects a prompt source rooted in fsys.
func (o *Options) SetPromptFS(fsys fs.FS, root string) error {
if err := validateSourceFS(fsys, root); err != nil {
return err
}
o.promptDefinitions = promptdef.NewFSRepository(fsys, root)
o.promptSource = true
return nil
}
// SetPromptFile selects a single prompt file.
func (o *Options) SetPromptFile(path string) error {
fsys, root, directory, err := fileSource(path)
if err != nil {
return err
}
o.promptDefinitions = promptdef.NewFileRepository(fsys, root, directory)
o.promptSource = true
return nil
}
// SetProfileFS selects an ordinary profile source rooted in fsys.
func (o *Options) SetProfileFS(fsys fs.FS, root string) error {
if err := validateSourceFS(fsys, root); err != nil {
return err
}
o.profiles = profile.NewFSRepository(fsys, root)
o.profileSource = true
return nil
}
// SetProfileFile selects a single ordinary profile file.
func (o *Options) SetProfileFile(path string) error {
fsys, root, _, err := fileSource(path)
if err != nil {
return err
}
o.profiles = profile.NewFSRepository(fsys, root)
o.profileSource = true
return nil
}
// SetFallbackProfileFS selects a fallback profile source rooted in fsys.
func (o *Options) SetFallbackProfileFS(fsys fs.FS, root string) error {
if err := validateSourceFS(fsys, root); err != nil {
return err
}
o.fallbackProfiles = profile.NewFSRepository(fsys, root)
o.fallbackProfileSource = true
return nil
}
// SetMemoryProfiles validates and stores application-owned profile definitions.
func (o *Options) SetMemoryProfiles(definitions []domain.ExecutionProfile) error {
repository, err := profile.NewMemoryRepository(definitions)
if err != nil {
return err
}
o.memoryProfiles = repository
o.memorySource = true
return nil
}
// AddBackend appends one backend registration.
func (o *Options) AddBackend(backend domain.Backend) { o.backends = append(o.backends, backend) }
// SetSchemaFS selects a schema source rooted in fsys.
func (o *Options) SetSchemaFS(fsys fs.FS, root string) error {
if err := validateSourceFS(fsys, root); err != nil {
@@ -163,33 +90,20 @@ func New(cfg Config, options Options) (*Engine, error) {
if err != nil {
return nil, err
}
promptDefinitions := options.promptDefinitions
if !options.promptSource {
if strings.TrimSpace(cfg.PromptDir) == "" {
return nil, fmt.Errorf("prompt directory is required")
}
promptDefinitions = promptdef.NewFilesystemRepository(cfg.PromptDir)
}
maintained, err := legacycatalog.Load(
legacycatalog.Source{Name: "OpenRouter", ExpectedBackendID: backend.OpenRouterID, FS: openrouter.FS(), Root: openrouter.Root},
legacycatalog.Source{Name: "Rakestrawhome", ExpectedBackendID: backend.RakestrawHomeID, FS: rakestrawhome.FS(), Root: rakestrawhome.Root},
)
underlays, err := catalog.Underlays()
if err != nil {
return nil, fmt.Errorf("failed to load maintained catalogs: %w", err)
return nil, fmt.Errorf("select maintained configuration: %w", err)
}
maintainedProfiles, err := newMaintainedProfileRepository(maintained.Profiles, maintained.Backends)
underlays = append(underlays, cfg.Sources.Underlays...)
main := configkit.Input{Source: configkit.NoSource()}
if cfg.Sources.Main != nil {
main = *cfg.Sources.Main
}
snapshot, err := configuration.Load(main, underlays)
if err != nil {
return nil, fmt.Errorf("failed to construct Promptkit-maintained profiles: %w", err)
return nil, err
}
profiles := newProfileRepository(cfg.ProfileDir, options, maintainedProfiles)
registry, err := backend.NewRegistry(maintained.Backends, options.backends)
if err != nil {
return nil, fmt.Errorf("failed to construct backend registry: %w", err)
}
manager, err := capacity.NewManager(registry.CapacityPolicies())
manager, err := capacity.NewManager(snapshot.CapacityPolicies())
if err != nil {
return nil, fmt.Errorf("failed to construct backend capacity manager: %w", err)
}
@@ -217,23 +131,7 @@ func New(cfg Config, options Options) (*Engine, error) {
reader = artifact.NewCompositeReader()
}
return &Engine{runner: usecase.NewRunnerWithRepairer(promptDefinitions, profiles, registry, reader, prompt.NewGoRenderer(), client, validator, usecase.NewDefaultOutputRepairer(client), manager, defaultRetries)}, nil
}
func newProfileRepository(profileDir string, options Options, maintained profile.Repository) profile.Repository {
repository := maintained
if options.fallbackProfileSource {
repository = profile.NewOverlayRepository(options.fallbackProfiles, repository)
}
if options.profileSource {
repository = profile.NewOverlayRepository(options.profiles, repository)
} else if strings.TrimSpace(profileDir) != "" {
repository = profile.NewOverlayRepository(profile.NewFilesystemRepository(profileDir), repository)
}
if options.memorySource {
repository = profile.NewOverlayRepository(options.memoryProfiles, repository)
}
return profile.NewResolvingRepository(repository)
return &Engine{runner: usecase.NewRunnerWithRepairer(snapshot, snapshot, snapshot, reader, prompt.NewGoRenderer(), client, validator, usecase.NewDefaultOutputRepairer(client), manager, defaultRetries)}, nil
}
func validateSourceFS(fsys fs.FS, root string) error {

18
internal/usecase/profile_inspection.go
View File

@@ -19,15 +19,18 @@ func (r *Runner) resolveProfileSelection(
ctx context.Context,
profileID string,
) (*resolvedProfileSelection, error) {
normalizedID := strings.TrimSpace(profileID)
if normalizedID == "" {
lookupID := profileID
if strings.TrimSpace(lookupID) == "" {
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
}
if r == nil || r.profiles == nil {
return nil, fmt.Errorf("%w: profile repository is not configured", ErrProfileLoad)
}
if exact, ok := r.profiles.(interface{ ExactIDs() bool }); !ok || !exact.ExactIDs() {
lookupID = strings.TrimSpace(lookupID)
}
selectedProfile, err := r.profiles.GetProfile(ctx, normalizedID)
selectedProfile, err := r.profiles.GetProfile(ctx, lookupID)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
}
@@ -37,7 +40,7 @@ func (r *Runner) resolveProfileSelection(
profileValue := *selectedProfile
if profileValue.ResolvedTarget != nil {
return &resolvedProfileSelection{id: normalizedID, profile: &profileValue}, nil
return &resolvedProfileSelection{id: lookupID, profile: &profileValue}, nil
}
profileValue.BackendID = strings.TrimSpace(profileValue.BackendID)
@@ -54,7 +57,7 @@ func (r *Runner) resolveProfileSelection(
}
return &resolvedProfileSelection{
id: normalizedID,
id: lookupID,
profile: &profileValue,
backend: selectedBackend,
}, nil
@@ -80,8 +83,7 @@ func (r *Runner) InspectProfile(
ctx context.Context,
profileID string,
) (*domain.ProfileInspection, error) {
normalizedID := strings.TrimSpace(profileID)
if normalizedID == "" {
if strings.TrimSpace(profileID) == "" {
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidRequest)
}
select {
@@ -90,7 +92,7 @@ func (r *Runner) InspectProfile(
default:
}
selection, err := r.resolveProfileSelection(ctx, normalizedID)
selection, err := r.resolveProfileSelection(ctx, profileID)
if err != nil {
return nil, err
}

15
internal/usecase/prompt_inspection.go
View File

@@ -25,7 +25,16 @@ func (r *Runner) resolvePromptDefinition(
return nil, fmt.Errorf("%w: prompt repository is not configured", ErrPromptLoad)
}
definition, err := r.promptDefs.GetPromptDefinition(ctx, promptID, promptVersion)
var definition *domain.PromptDefinition
var hash string
var err error
if indexed, ok := r.promptDefs.(interface {
GetPromptSnapshot(context.Context, string, string) (*domain.PromptDefinition, string, error)
}); ok {
definition, hash, err = indexed.GetPromptSnapshot(ctx, promptID, promptVersion)
} else {
definition, err = r.promptDefs.GetPromptDefinition(ctx, promptID, promptVersion)
}
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrPromptLoad, err)
}
@@ -33,10 +42,12 @@ func (r *Runner) resolvePromptDefinition(
return nil, fmt.Errorf("%w: prompt repository returned nil definition", ErrPromptLoad)
}
hash, err := hashPromptDefinition(definition)
if hash == "" {
hash, err = hashPromptDefinition(definition)
if err != nil {
return nil, fmt.Errorf("%w: failed to hash prompt definition: %v", ErrPromptLoad, err)
}
}
return &resolvedPromptDefinition{definition: definition, hash: hash}, nil
}

6
internal/usecase/runner.go
View File

@@ -339,9 +339,9 @@ func (r *Runner) resolvePreparation(
return nil, fmt.Errorf("%w: output contract: %v", ErrInvalidRequest, err)
}
selectedProfileID := strings.TrimSpace(req.ProfileID)
if selectedProfileID == "" {
selectedProfileID = strings.TrimSpace(def.DefaultProfile)
selectedProfileID := req.ProfileID
if strings.TrimSpace(selectedProfileID) == "" {
selectedProfileID = def.DefaultProfile
}
if selectedProfileID == "" {
return nil, fmt.Errorf("%w: %w: profile id is required either in request or prompt default_profile", ErrInvalidRequest, ErrProfileRequired)

7
observation_boundary_contract_test.go
View File

@@ -8,6 +8,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/promptkit"
"gitea.maximumdirect.net/eric/promptkit/testfixture"
)
type observationClientFunc func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error)
@@ -23,9 +24,9 @@ func TestDelegatedOperationsIsolateObservations(t *testing.T) {
ctx, cancel := context.WithTimeout(context.WithValue(context.Background(), contextKey{}, "caller-value"), time.Minute)
defer cancel()
deadline, _ := ctx.Deadline()
child, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("child", "profile", "child message"), "."),
promptkit.WithProfiles(promptkit.Profile{ID: "profile", Endpoint: "https://example.invalid/v1", Model: "child-model"}),
child, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("child", "profile", "child message"), "."),
testfixture.WithProfiles(testfixture.Profile{ID: "profile", Endpoint: "https://example.invalid/v1", Model: "child-model"}),
promptkit.WithLLMClient(observationClientFunc(func(callCtx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
gotDeadline, ok := callCtx.Deadline()
if !ok || gotDeadline != deadline || callCtx.Value(contextKey{}) != "caller-value" || callCtx.Done() != ctx.Done() {

9
observation_contract_test.go
View File

@@ -9,13 +9,14 @@ import (
"testing/fstest"
"gitea.maximumdirect.net/eric/promptkit"
"gitea.maximumdirect.net/eric/promptkit/testfixture"
)
func observationEngine(t *testing.T, client promptkit.LLMClient) *promptkit.Engine {
t.Helper()
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "hello"), "."),
promptkit.WithProfiles(promptkit.Profile{ID: "profile", Endpoint: "https://example.invalid/v1", Model: "model"}),
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "hello"), "."),
testfixture.WithProfiles(testfixture.Profile{ID: "profile", Endpoint: "https://example.invalid/v1", Model: "model"}),
promptkit.WithLLMClient(client))
if err != nil {
t.Fatal(err)
@@ -128,7 +129,7 @@ func TestRunObservationClassifiesInvalidOutputAndGenerationFailure(t *testing.T)
}
func TestRunObservationPromptFailureAndPanicCleanup(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithPromptFS(fstest.MapFS{}, "."))
engine, err := testfixture.NewEngine(testfixture.Config{}, testfixture.WithPromptFS(fstest.MapFS{}, "."))
if err != nil {
t.Fatal(err)
}

9
output_contract_contract_test.go
View File

@@ -6,13 +6,14 @@ import (
"testing"
"gitea.maximumdirect.net/eric/promptkit"
"gitea.maximumdirect.net/eric/promptkit/testfixture"
)
func TestPreparationRejectsInvalidOutputContractWithPublicError(t *testing.T) {
engine, err := promptkit.NewEngine(
promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
promptkit.WithProfiles(promptkit.Profile{
engine, err := testfixture.NewEngine(
testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
testfixture.WithProfiles(testfixture.Profile{
ID: "profile",
Endpoint: "http://example.test/v1",
Model: "model",

33
prepared_execution_contract_test.go
View File

@@ -13,6 +13,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/promptkit"
"gitea.maximumdirect.net/eric/promptkit/testfixture"
)
func TestPreparedExecutionFreezesSourcesAndReturnsIndependentDetails(t *testing.T) {
@@ -26,10 +27,10 @@ func TestPreparedExecutionFreezesSourcesAndReturnsIndependentDetails(t *testing.
client := &preparedRecordingClient{
response: &promptkit.GenerateResponse{Content: `{"value":3}`},
}
engine, err := promptkit.NewEngine(
promptkit.Config{},
promptkit.WithPromptFS(promptSource, "."),
promptkit.WithProfileFS(profileSource, "."),
engine, err := testfixture.NewEngine(
testfixture.Config{},
testfixture.WithPromptFS(promptSource, "."),
testfixture.WithProfileFS(profileSource, "."),
promptkit.WithSchemaFS(schemaSource, "."),
promptkit.WithArtifactReader(reader),
promptkit.WithLLMClient(client),
@@ -430,10 +431,10 @@ func TestPreparedExecutionDiscardAndFormattingDoNotExposePrivateState(t *testing
client := &preparedRecordingClient{
response: &promptkit.GenerateResponse{Content: "generated output"},
}
engine, err := promptkit.NewEngine(
promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prepared", "profile", renderedContent), "."),
promptkit.WithProfiles(promptkit.Profile{
engine, err := testfixture.NewEngine(
testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prepared", "profile", renderedContent), "."),
testfixture.WithProfiles(testfixture.Profile{
ID: "profile",
Endpoint: "http://example.test/v1",
Model: "model",
@@ -564,10 +565,10 @@ func TestPreparedExecutionCredentialCapacityAndTimingBoundaries(t *testing.T) {
client := &preparedRecordingClient{
response: &promptkit.GenerateResponse{Content: "unexpected"},
}
engine, err := promptkit.NewEngine(
promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prepared", "profile", "content"), "."),
promptkit.WithProfiles(promptkit.Profile{
engine, err := testfixture.NewEngine(
testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prepared", "profile", "content"), "."),
testfixture.WithProfiles(testfixture.Profile{
ID: "profile",
Endpoint: "http://example.test/v1",
Model: "model",
@@ -766,10 +767,10 @@ func newPreparedContractEngine(
message string,
) *promptkit.Engine {
t.Helper()
engine, err := promptkit.NewEngine(
promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prepared", "profile", message), "."),
promptkit.WithProfiles(promptkit.Profile{
engine, err := testfixture.NewEngine(
testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prepared", "profile", message), "."),
testfixture.WithProfiles(testfixture.Profile{
ID: "profile",
Endpoint: "http://example.test/v1",
Model: "model",

7
prepared_observation_contract_test.go
View File

@@ -7,6 +7,7 @@ import (
"testing"
"gitea.maximumdirect.net/eric/promptkit"
"gitea.maximumdirect.net/eric/promptkit/testfixture"
)
func TestPreparedObservationStartsOnlyAfterClaimAndKeepsObserver(t *testing.T) {
@@ -131,9 +132,9 @@ func TestPreparedObservationClaimRaceAndPanicCleanup(t *testing.T) {
func TestPreparedObservationFinishesAfterCredentialAndCapacityFailures(t *testing.T) {
const envName = "PROMPTKIT_PREPARED_OBSERVATION_KEY"
t.Setenv(envName, "available")
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prepared", "profile", "hello"), "."),
promptkit.WithProfiles(promptkit.Profile{ID: "profile", Endpoint: "https://example.invalid/v1", Model: "model", APIKeyRequired: true}),
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prepared", "profile", "hello"), "."),
testfixture.WithProfiles(testfixture.Profile{ID: "profile", Endpoint: "https://example.invalid/v1", Model: "model", APIKeyRequired: true}),
promptkit.WithLLMClient(&fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}))
if err != nil {
t.Fatal(err)

57
profile_inheritance_contract_test.go
View File

@@ -10,13 +10,14 @@ import (
"testing/fstest"
"gitea.maximumdirect.net/eric/promptkit"
"gitea.maximumdirect.net/eric/promptkit/testfixture"
)
func TestProfileInheritanceBuiltInAliasWorkflow(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{
engine, err := testfixture.NewEngine(testfixture.Config{
PromptDir: frameworkPromptDir,
SchemaDir: frameworkSchemaDir,
}, promptkit.WithProfiles(promptkit.Profile{
}, testfixture.WithProfiles(testfixture.Profile{
ID: "weather-light",
BaseProfileID: "promptkit-openrouter-standard",
ReasoningEffort: "high",
@@ -81,9 +82,9 @@ func TestProfileInheritanceBuiltInAliasWorkflow(t *testing.T) {
}
func TestProfileInheritanceYAMLAliasOfBuiltIn(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(fstest.MapFS{}, "."),
promptkit.WithProfileFS(fstest.MapFS{
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(fstest.MapFS{}, "."),
testfixture.WithProfileFS(fstest.MapFS{
"alias.yaml": &fstest.MapFile{Data: []byte("id: yaml-alias\nbase_profile: promptkit-openrouter-standard\n")},
}, "."),
)
@@ -100,21 +101,21 @@ func TestProfileInheritanceYAMLAliasOfBuiltIn(t *testing.T) {
}
func TestProfileInheritanceRetainsRequiredCredentialBehavior(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(fstest.MapFS{}, "."),
promptkit.WithBackend(promptkit.Backend{
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(fstest.MapFS{}, "."),
testfixture.WithBackend(testfixture.Backend{
ID: "credential-backend",
Endpoint: "https://credential.example/v1",
APIKeyEnv: "OPTIONAL_BACKEND_KEY",
}),
promptkit.WithProfiles(
promptkit.Profile{
testfixture.WithProfiles(
testfixture.Profile{
ID: "credential-base",
BackendID: "credential-backend",
Model: "model",
APIKeyRequired: true,
},
promptkit.Profile{ID: "credential-child", BaseProfileID: "credential-base"},
testfixture.Profile{ID: "credential-child", BaseProfileID: "credential-base"},
),
)
if err != nil {
@@ -130,17 +131,13 @@ func TestProfileInheritanceRetainsRequiredCredentialBehavior(t *testing.T) {
}
func TestProfileInheritancePreservesPublicErrorIdentities(t *testing.T) {
newEngine := func(t *testing.T, profiles fs.FS) *promptkit.Engine {
newEngine := func(t *testing.T, profiles fs.FS) (*promptkit.Engine, error) {
t.Helper()
options := []promptkit.Option{promptkit.WithPromptFS(fstest.MapFS{}, ".")}
options := []any{testfixture.WithPromptFS(fstest.MapFS{}, ".")}
if profiles != nil {
options = append(options, promptkit.WithProfileFS(profiles, "."))
options = append(options, testfixture.WithProfileFS(profiles, "."))
}
engine, err := promptkit.NewEngine(promptkit.Config{}, options...)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
return engine
return testfixture.NewEngine(testfixture.Config{}, options...)
}
tests := []struct {
@@ -181,7 +178,17 @@ func TestProfileInheritancePreservesPublicErrorIdentities(t *testing.T) {
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
result, err := newEngine(t, tc.profiles).InspectProfile(context.Background(), tc.profile)
engine, err := newEngine(t, tc.profiles)
if tc.profiles != nil {
if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("construction = (%#v, %v), want invalid configuration", engine, err)
}
return
}
if err != nil {
t.Fatalf("construct engine: %v", err)
}
result, err := engine.InspectProfile(context.Background(), tc.profile)
if result != nil || !errors.Is(err, tc.want) || (tc.wantNot != nil && errors.Is(err, tc.wantNot)) {
t.Fatalf("inspection=(%+v, %v), want %v without %v", result, err, tc.want, tc.wantNot)
}
@@ -200,9 +207,9 @@ func TestProfileInheritanceFreezesPreparedExecution(t *testing.T) {
"base.yaml": &fstest.MapFile{Data: []byte("id: base\nendpoint: https://base.example/v1\nmodel: first-model\n")},
}}
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prepared", "child", "content"), "."),
promptkit.WithProfileFS(profiles, "."),
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prepared", "child", "content"), "."),
testfixture.WithProfileFS(profiles, "."),
promptkit.WithLLMClient(client),
)
if err != nil {
@@ -220,8 +227,8 @@ func TestProfileInheritanceFreezesPreparedExecution(t *testing.T) {
t.Fatalf("prepared execution=(%+v, %v), requests=%+v", result, err, client.requests)
}
inspection, err := engine.InspectProfile(context.Background(), "child")
if err != nil || inspection.EffectiveModelParams.Model != "second-model" {
t.Fatalf("fresh inspection=(%+v, %v)", inspection, err)
if err != nil || inspection.EffectiveModelParams.Model != "first-model" {
t.Fatalf("existing engine inspection=(%+v, %v)", inspection, err)
}
}

115
profiles.go
View File

@@ -1,64 +1,71 @@
package promptkit
import (
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
import "gitea.maximumdirect.net/eric/ConfigKit"
// OpenAICompatibleProfile returns an in-memory Profile for an OpenAI-compatible
// chat-completions endpoint. A non-blank BaseProfileID permits its target
// fields to be inherited when the profile is selected or inspected.
//
// It does not register global state, maintain a model catalog, or resolve
// credentials. If APIKeyRequired is true, callers satisfy it with
// RunRequest.APIKey or an explicit request ExecutionTargetOverride.APIKeyEnv.
// Raw API keys do not belong in profiles.
//
// The function copies the ExtraParams map itself but does not recursively copy
// nested values. Validation and a deep copy occur when NewEngine applies a
// WithProfiles option containing the returned Profile.
func OpenAICompatibleProfile(cfg OpenAICompatibleProfileConfig) Profile {
return Profile{
ID: cfg.ID,
BaseProfileID: cfg.BaseProfileID,
BackendID: cfg.BackendID,
Endpoint: cfg.Endpoint,
Model: cfg.Model,
Temperature: cfg.Temperature,
MaxTokens: cfg.MaxTokens,
TopP: cfg.TopP,
TimeoutSeconds: cfg.TimeoutSeconds,
ServiceTier: cfg.ServiceTier,
ReasoningEffort: cfg.ReasoningEffort,
APIKeyRequired: cfg.APIKeyRequired,
ExtraParams: copyShallowAnyMap(cfg.ExtraParams),
}
// ProfileConfig is a presence-aware convenience value for a keyed execution
// profile. Nil fields are omitted; non-nil pointers preserve zero and blank
// values. A nil ExtraParams map is omitted; a non-nil empty map supplies an
// empty mapping, not a clear operation. Raw ConfigKit Mapping inputs can
// express explicit null. ProfileConfig has no stable JSON representation.
type ProfileConfig struct {
Extends, Backend, Endpoint, Model *string
Temperature, TopP *float64
MaxTokens, TimeoutSeconds *int
ServiceTier, ReasoningEffort *string
APIKeyEnv *string
APIKeyRequired *bool
ExtraParams map[string]any
}
func copyShallowAnyMap(src map[string]any) map[string]any {
if src == nil {
return nil
// ProfileSource creates a labeled ConfigKit mapping from keyed profile values.
// It encodes fields without validating them; NewEngine validates the final
// effective configuration after all selected source layers are combined.
// Mapping captures a copy of supported input values at creation; unsupported
// values fail when the source is loaded.
func ProfileSource(label string, profiles map[string]ProfileConfig) configkit.Source {
definitions := make(map[string]any, len(profiles))
for id, config := range profiles {
fields := make(map[string]any)
if config.Extends != nil {
fields["extends"] = *config.Extends
}
out := make(map[string]any, len(src))
for k, v := range src {
out[k] = v
if config.Backend != nil {
fields["backend"] = *config.Backend
}
return out
if config.Endpoint != nil {
fields["endpoint"] = *config.Endpoint
}
func toDomainProfile(publicProfile Profile) domain.ExecutionProfile {
return domain.ExecutionProfile{
ID: publicProfile.ID,
BaseProfileID: publicProfile.BaseProfileID,
BackendID: publicProfile.BackendID,
Endpoint: publicProfile.Endpoint,
Model: publicProfile.Model,
Temperature: publicProfile.Temperature,
MaxTokens: publicProfile.MaxTokens,
TopP: publicProfile.TopP,
TimeoutSeconds: publicProfile.TimeoutSeconds,
ServiceTier: publicProfile.ServiceTier,
ReasoningEffort: publicProfile.ReasoningEffort,
APIKeyRequired: publicProfile.APIKeyRequired,
ExtraParams: publicProfile.ExtraParams,
if config.Model != nil {
fields["model"] = *config.Model
}
if config.Temperature != nil {
fields["temperature"] = *config.Temperature
}
if config.TopP != nil {
fields["top_p"] = *config.TopP
}
if config.MaxTokens != nil {
fields["max_tokens"] = *config.MaxTokens
}
if config.TimeoutSeconds != nil {
fields["timeout_seconds"] = *config.TimeoutSeconds
}
if config.ServiceTier != nil {
fields["service_tier"] = *config.ServiceTier
}
if config.ReasoningEffort != nil {
fields["reasoning_effort"] = *config.ReasoningEffort
}
if config.APIKeyEnv != nil {
fields["api_key_env"] = *config.APIKeyEnv
}
if config.APIKeyRequired != nil {
fields["api_key_required"] = *config.APIKeyRequired
}
if config.ExtraParams != nil {
fields["extra_params"] = providerMapForMapping(config.ExtraParams)
}
definitions[id] = fields
}
return configkit.Mapping(label, map[string]any{"profiles": definitions})
}

121
provider_values.go Normal file
View File

@@ -0,0 +1,121 @@
package promptkit
import (
"encoding/json"
"reflect"
"strconv"
)
type providerVisit struct {
typ reflect.Type
ptr uintptr
}
type invalidProviderNumber struct{}
// providerMapForMapping converts JSON-shaped Go containers to ConfigKit's
// built-in mapping containers. Unsupported values remain for Mapping to report
// as deferred source diagnostics during construction.
func providerMapForMapping(values map[string]any) map[string]any {
if values == nil {
return nil
}
converted := make(map[string]any, len(values))
active := make(map[providerVisit]bool)
for key, value := range values {
converted[key] = providerValueForMapping(reflect.ValueOf(value), active, 0)
}
return converted
}
func providerValueForMapping(value reflect.Value, active map[providerVisit]bool, depth int) any {
if !value.IsValid() {
return nil
}
if depth > 100 {
return value.Interface()
}
for value.Kind() == reflect.Interface || value.Kind() == reflect.Pointer {
if value.IsNil() {
return nil
}
if value.Kind() == reflect.Pointer {
visit := providerVisit{typ: value.Type(), ptr: value.Pointer()}
if active[visit] {
return value.Interface()
}
active[visit] = true
defer delete(active, visit)
}
value = value.Elem()
}
if value.CanInterface() {
if number, ok := value.Interface().(json.Number); ok {
if _, err := json.Marshal(number); err != nil {
return invalidProviderNumber{}
}
if integer, err := strconv.ParseInt(string(number), 10, 64); err == nil {
return integer
}
if integer, err := strconv.ParseUint(string(number), 10, 64); err == nil {
return integer
}
if decimal, err := strconv.ParseFloat(string(number), 64); err == nil {
return decimal
}
return number
}
}
switch value.Kind() {
case reflect.Map:
if value.IsNil() {
return nil
}
if value.Type().Key().Kind() != reflect.String {
return value.Interface()
}
visit := providerVisit{typ: value.Type(), ptr: value.Pointer()}
if active[visit] {
return value.Interface()
}
active[visit] = true
defer delete(active, visit)
converted := make(map[string]any, value.Len())
iter := value.MapRange()
for iter.Next() {
converted[iter.Key().String()] = providerValueForMapping(iter.Value(), active, depth+1)
}
return converted
case reflect.Slice, reflect.Array:
if value.Kind() == reflect.Slice {
if value.IsNil() {
return nil
}
visit := providerVisit{typ: value.Type(), ptr: value.Pointer()}
if visit.ptr != 0 {
if active[visit] {
return value.Interface()
}
active[visit] = true
defer delete(active, visit)
}
}
converted := make([]any, value.Len())
for index := range converted {
converted[index] = providerValueForMapping(value.Index(index), active, depth+1)
}
return converted
case reflect.String:
return value.String()
case reflect.Bool:
return value.Bool()
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
return value.Int()
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
return value.Uint()
case reflect.Float32, reflect.Float64:
return value.Float()
default:
return value.Interface()
}
}

433
public_contract_test.go
View File

@@ -5,7 +5,6 @@ import (
"encoding/json"
"errors"
"fmt"
"io/fs"
"reflect"
"strings"
"sync"
@@ -14,33 +13,25 @@ import (
"testing/fstest"
"gitea.maximumdirect.net/eric/promptkit"
"gitea.maximumdirect.net/eric/promptkit/testfixture"
)
type inspectionCountingFS struct {
opens atomic.Int64
}
func (f *inspectionCountingFS) Open(string) (fs.File, error) {
f.opens.Add(1)
return nil, fs.ErrNotExist
}
func TestInspectProfileResolvesCredentialStatesWithoutPromptOrGeneration(t *testing.T) {
const environmentName = "PROMPTKIT_INSPECTION_ABSENT_KEY"
t.Setenv(environmentName, "")
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}}
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(fstest.MapFS{}, "."),
promptkit.WithProfileFS(fstest.MapFS{
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(fstest.MapFS{}, "."),
testfixture.WithProfileFS(fstest.MapFS{
"environment.yaml": &fstest.MapFile{Data: []byte(`id: environment
endpoint: http://environment.example/v1
model: environment-model
api_key_env: PROMPTKIT_INSPECTION_ABSENT_KEY
`)},
}, "."),
promptkit.WithProfiles(
promptkit.Profile{ID: "direct", Endpoint: "http://direct.example/v1", Model: "direct-model", APIKeyRequired: true},
promptkit.Profile{ID: "none", Endpoint: "http://none.example/v1", Model: "none-model"},
testfixture.WithProfiles(
testfixture.Profile{ID: "direct", Endpoint: "http://direct.example/v1", Model: "direct-model", APIKeyRequired: true},
testfixture.Profile{ID: "none", Endpoint: "http://none.example/v1", Model: "none-model"},
),
promptkit.WithLLMClient(client),
)
@@ -63,7 +54,7 @@ api_key_env: PROMPTKIT_INSPECTION_ABSENT_KEY
if err != nil {
t.Fatalf("inspect profile: %v", err)
}
if inspection.ProfileID != strings.TrimSpace(tc.profileID) ||
if inspection.ProfileID != tc.profileID ||
inspection.EffectiveModelParams.Endpoint != tc.wantEndpoint ||
inspection.EffectiveModelParams.BackendID != "" ||
inspection.EffectiveModelParams.APIKeyEnv != tc.wantEnv ||
@@ -77,10 +68,10 @@ api_key_env: PROMPTKIT_INSPECTION_ABSENT_KEY
}
}
func TestFileProfileNormalizedIDMatchesInspectionAndPreparation(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "normalized-profile", "message"), "."),
promptkit.WithProfileFS(fstest.MapFS{
func TestFileProfileExactIDMatchesInspectionAndPreparation(t *testing.T) {
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", " normalized-profile ", "message"), "."),
testfixture.WithProfileFS(fstest.MapFS{
"profile.yaml": &fstest.MapFile{Data: []byte(`
id: " normalized-profile "
endpoint: http://profile.example/v1
@@ -113,11 +104,11 @@ model: normalized-model
}
func TestInspectProfilePreservesPublicErrorIdentities(t *testing.T) {
newEngine := func(t *testing.T, options ...promptkit.Option) *promptkit.Engine {
newEngine := func(t *testing.T, options ...any) *promptkit.Engine {
t.Helper()
engine, err := promptkit.NewEngine(
promptkit.Config{},
append([]promptkit.Option{promptkit.WithPromptFS(fstest.MapFS{}, ".")}, options...)...,
engine, err := testfixture.NewEngine(
testfixture.Config{},
append([]any{testfixture.WithPromptFS(fstest.MapFS{}, ".")}, options...)...,
)
if err != nil {
t.Fatalf("construct inspection engine: %v", err)
@@ -131,7 +122,7 @@ func TestInspectProfilePreservesPublicErrorIdentities(t *testing.T) {
t.Fatalf("nil engine result=(%#v, %v), want ErrInvalidConfig", result, err)
}
valid := newEngine(t, promptkit.WithProfiles(promptkit.Profile{
valid := newEngine(t, testfixture.WithProfiles(testfixture.Profile{
ID: "profile", Endpoint: "http://profile.example/v1", Model: "model",
}))
if result, err := valid.InspectProfile(context.Background(), " \t "); result != nil ||
@@ -143,29 +134,25 @@ func TestInspectProfilePreservesPublicErrorIdentities(t *testing.T) {
t.Fatalf("missing profile result=(%#v, %v), want only ErrProfileNotFound", result, err)
}
malformed := newEngine(t, promptkit.WithProfileFS(fstest.MapFS{
malformed, err := testfixture.NewEngine(testfixture.Config{}, testfixture.WithProfileFS(fstest.MapFS{
"broken.yaml": &fstest.MapFile{Data: []byte("id: broken\nendpoint: http://broken.example/v1\nmodel: model\nextra_params:\n invalid: .nan\n")},
}, "."))
if result, err := malformed.InspectProfile(context.Background(), "broken"); result != nil ||
!errors.Is(err, promptkit.ErrProfileLoad) {
t.Fatalf("malformed profile result=(%#v, %v), want ErrProfileLoad", result, err)
if malformed != nil || !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("malformed profile construction=(%#v, %v), want ErrInvalidConfig", malformed, err)
}
unknownBackend := newEngine(t, promptkit.WithProfiles(promptkit.Profile{
unknownBackend, err := testfixture.NewEngine(testfixture.Config{}, testfixture.WithProfiles(testfixture.Profile{
ID: "unknown-backend", BackendID: "unknown", Model: "model",
}))
if result, err := unknownBackend.InspectProfile(context.Background(), "unknown-backend"); result != nil ||
!errors.Is(err, promptkit.ErrProfileLoad) {
t.Fatalf("unknown backend result=(%#v, %v), want ErrProfileLoad", result, err)
if unknownBackend != nil || !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("unknown backend construction=(%#v, %v), want ErrInvalidConfig", unknownBackend, err)
}
countingFS := &inspectionCountingFS{}
canceled := newEngine(t, promptkit.WithProfileFS(countingFS, "."))
canceled := newEngine(t, testfixture.WithProfiles(testfixture.Profile{ID: "profile", Endpoint: "http://profile.example/v1"}))
ctx, cancel := context.WithCancel(context.Background())
cancel()
if result, err := canceled.InspectProfile(ctx, "profile"); result != nil ||
!errors.Is(err, promptkit.ErrProfileLoad) || !errors.Is(err, context.Canceled) || countingFS.opens.Load() != 0 {
t.Fatalf("canceled inspection result=(%#v, %v), opens=%d", result, err, countingFS.opens.Load())
if result, err := canceled.InspectProfile(ctx, "profile"); result != nil || !errors.Is(err, context.Canceled) {
t.Fatalf("canceled inspection result=(%#v, %v)", result, err)
}
}
@@ -173,9 +160,9 @@ func TestInspectProfileReturnsIndependentTargetMatchingPreparation(t *testing.T)
extraParams := map[string]any{
"nested": map[string]any{"value": "original"},
}
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
promptkit.WithProfiles(promptkit.Profile{
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
testfixture.WithProfiles(testfixture.Profile{
ID: "profile", Endpoint: "http://profile.example/v1", Model: "model", ExtraParams: extraParams,
}),
)
@@ -210,8 +197,8 @@ func TestInspectProfileReturnsIndependentTargetMatchingPreparation(t *testing.T)
func TestInspectPromptReturnsDeclaredMetadataWithoutExecutionWork(t *testing.T) {
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}}
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(fstest.MapFS{
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(fstest.MapFS{
"report-v1.yaml": &fstest.MapFile{Data: []byte(`id: report
version: "1.0.0"
messages:
@@ -278,7 +265,7 @@ output:
func TestInspectPromptPreservesPublicErrorIdentities(t *testing.T) {
newEngine := func(t *testing.T, source fstest.MapFS) *promptkit.Engine {
t.Helper()
engine, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithPromptFS(source, "."))
engine, err := testfixture.NewEngine(testfixture.Config{}, testfixture.WithPromptFS(source, "."))
if err != nil {
t.Fatalf("construct prompt-inspection engine: %v", err)
}
@@ -358,29 +345,25 @@ output:
},
} {
t.Run(name, func(t *testing.T) {
if result, err := newEngine(t, source).InspectPrompt(context.Background(), "broken", "1"); result != nil ||
!errors.Is(err, promptkit.ErrPromptLoad) {
t.Fatalf("broken prompt result=(%#v, %v), want ErrPromptLoad", result, err)
engine, err := testfixture.NewEngine(testfixture.Config{}, testfixture.WithPromptFS(source, "."))
if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("broken prompt construction=(%#v, %v), want ErrInvalidConfig", engine, err)
}
})
}
countingFS := &inspectionCountingFS{}
canceled, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithPromptFS(countingFS, "."))
if err != nil {
t.Fatalf("construct canceled prompt-inspection engine: %v", err)
}
canceled := valid
ctx, cancel := context.WithCancel(context.Background())
cancel()
if result, err := canceled.InspectPrompt(ctx, "prompt", "1"); result != nil ||
!errors.Is(err, promptkit.ErrPromptLoad) || !errors.Is(err, context.Canceled) || countingFS.opens.Load() != 0 {
t.Fatalf("canceled inspection result=(%#v, %v), opens=%d", result, err, countingFS.opens.Load())
!errors.Is(err, context.Canceled) {
t.Fatalf("canceled inspection result=(%#v, %v)", result, err)
}
}
func TestInspectPromptReturnsIndependentMetadataMatchingPreparation(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(fstest.MapFS{
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(fstest.MapFS{
"prompt.yaml": &fstest.MapFile{Data: []byte(`id: prompt
version: "1"
default_profile: profile
@@ -396,7 +379,7 @@ output:
validation_mode: basic
`)},
}, "."),
promptkit.WithProfiles(promptkit.Profile{
testfixture.WithProfiles(testfixture.Profile{
ID: "profile", Endpoint: "http://profile.example/v1", Model: "model",
}),
)
@@ -482,9 +465,9 @@ func TestBackendIdentityJSONNamesAndOmission(t *testing.T) {
func TestEndpointOnlyProfileOmitsBackendIdentityFromStableJSON(t *testing.T) {
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
promptkit.WithProfiles(promptkit.Profile{
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
testfixture.WithProfiles(testfixture.Profile{
ID: "profile", Endpoint: "http://example.test/v1", Model: "model",
}),
promptkit.WithLLMClient(client),
@@ -518,21 +501,13 @@ func TestEndpointOnlyProfileOmitsBackendIdentityFromStableJSON(t *testing.T) {
}
}
func TestUnknownProfileBackendHasProfileLoadIdentity(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
promptkit.WithProfiles(promptkit.Profile{ID: "profile", BackendID: "unknown", Model: "model"}),
func TestUnknownProfileBackendFailsConstruction(t *testing.T) {
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
testfixture.WithProfiles(testfixture.Profile{ID: "profile", BackendID: "unknown", Model: "model"}),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
_, err = engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if !errors.Is(err, promptkit.ErrProfileLoad) {
t.Fatalf("expected ErrProfileLoad, got %v", err)
}
if errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("unknown backend should not have invalid-request identity: %v", err)
if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("construction=(%#v, %v), want ErrInvalidConfig", engine, err)
}
}
@@ -546,8 +521,8 @@ func TestLocalBackendConstructsAndRegistersConventionalBackend(t *testing.T) {
}
endpoint := "http://local.example/v1"
backend := promptkit.LocalBackend(endpoint, limit)
want := promptkit.Backend{
backend := testfixture.LocalBackend(endpoint, limit)
want := testfixture.Backend{
ID: localBackendID,
Endpoint: endpoint,
ConcurrencyLimit: limit,
@@ -556,10 +531,10 @@ func TestLocalBackendConstructsAndRegistersConventionalBackend(t *testing.T) {
t.Fatalf("LocalBackend()=%+v, want %+v", backend, want)
}
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "local-profile", "message"), "."),
promptkit.WithBackend(backend),
promptkit.WithProfiles(promptkit.Profile{
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", "local-profile", "message"), "."),
testfixture.WithBackend(backend),
testfixture.WithProfiles(testfixture.Profile{
ID: "local-profile",
BackendID: localBackendID,
Model: "model",
@@ -582,9 +557,9 @@ func TestLocalBackendConstructsAndRegistersConventionalBackend(t *testing.T) {
func TestCustomBackendFlowsThroughProfilesOverridesAndInjectedClient(t *testing.T) {
t.Setenv("CUSTOM_LLM_KEY", "test-key")
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "backend-profile", "message"), "."),
promptkit.WithBackend(promptkit.Backend{
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", "backend-profile", "message"), "."),
testfixture.WithBackend(testfixture.Backend{
ID: "custom",
Endpoint: " http://backend.example/v1 ",
APIKeyEnv: " CUSTOM_LLM_KEY ",
@@ -592,10 +567,10 @@ func TestCustomBackendFlowsThroughProfilesOverridesAndInjectedClient(t *testing.
"provider": "custom",
},
}),
promptkit.WithProfiles(
promptkit.Profile{ID: "backend-profile", BackendID: "custom", Model: "backend-model"},
promptkit.Profile{ID: "profile-endpoint", BackendID: "custom", Endpoint: "http://profile.example/v1", Model: "profile-model"},
promptkit.Profile{ID: "blank-profile-endpoint", BackendID: "custom", Endpoint: " \t ", Model: "profile-model"},
testfixture.WithProfiles(
testfixture.Profile{ID: "backend-profile", BackendID: "custom", Model: "backend-model"},
testfixture.Profile{ID: "profile-endpoint", BackendID: "custom", Endpoint: "http://profile.example/v1", Model: "profile-model"},
testfixture.Profile{ID: "blank-profile-endpoint", BackendID: "custom", Model: "profile-model"},
),
promptkit.WithLLMClient(client),
)
@@ -655,16 +630,15 @@ func TestCustomBackendFlowsThroughProfilesOverridesAndInjectedClient(t *testing.
}
func TestCustomBackendSupportsFileProfileAndBothSelectionPaths(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "file-profile", "message"), "."),
promptkit.WithProfileFS(fstest.MapFS{
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", "file-profile", "message"), "."),
testfixture.WithProfileFS(fstest.MapFS{
"profile.yaml": &fstest.MapFile{Data: []byte(`id: file-profile
backend: file-backend
endpoint: " "
model: file-model
`)},
}, "."),
promptkit.WithBackend(promptkit.Backend{
testfixture.WithBackend(testfixture.Backend{
ID: "file-backend",
Endpoint: "http://file-backend.example/v1",
}),
@@ -689,13 +663,13 @@ model: file-model
}
func TestBackendOptionsAccumulateAndRegistrationsAreEngineLocal(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "first-profile", "message"), "."),
promptkit.WithBackend(promptkit.Backend{ID: "first", Endpoint: "http://first.example/v1"}),
promptkit.WithBackend(promptkit.Backend{ID: "second", Endpoint: "http://second.example/v1"}),
promptkit.WithProfiles(
promptkit.Profile{ID: "first-profile", BackendID: "first", Model: "model"},
promptkit.Profile{ID: "second-profile", BackendID: "second", Model: "model"},
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", "first-profile", "message"), "."),
testfixture.WithBackend(testfixture.Backend{ID: "first", Endpoint: "http://first.example/v1"}),
testfixture.WithBackend(testfixture.Backend{ID: "second", Endpoint: "http://second.example/v1"}),
testfixture.WithProfiles(
testfixture.Profile{ID: "first-profile", BackendID: "first", Model: "model"},
testfixture.Profile{ID: "second-profile", BackendID: "second", Model: "model"},
),
)
if err != nil {
@@ -718,10 +692,10 @@ func TestBackendOptionsAccumulateAndRegistrationsAreEngineLocal(t *testing.T) {
newEngine := func(endpoint string) *promptkit.Engine {
t.Helper()
value, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
promptkit.WithBackend(promptkit.Backend{ID: "same-id", Endpoint: endpoint}),
promptkit.WithProfiles(promptkit.Profile{ID: "profile", BackendID: "same-id", Model: "model"}),
value, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
testfixture.WithBackend(testfixture.Backend{ID: "same-id", Endpoint: endpoint}),
testfixture.WithProfiles(testfixture.Profile{ID: "profile", BackendID: "same-id", Model: "model"}),
)
if err != nil {
t.Fatalf("construct isolated engine: %v", err)
@@ -746,7 +720,7 @@ func TestBackendOptionsAccumulateAndRegistrationsAreEngineLocal(t *testing.T) {
func TestWithBackendCopiesQueueCapacity(t *testing.T) {
queueCapacity := 4
option := promptkit.WithBackend(promptkit.Backend{
option := testfixture.WithBackend(testfixture.Backend{
ID: "custom",
Endpoint: "http://custom.example/v1",
ConcurrencyLimit: 1,
@@ -754,10 +728,10 @@ func TestWithBackendCopiesQueueCapacity(t *testing.T) {
})
queueCapacity = -1
_, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
_, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
option,
promptkit.WithProfiles(promptkit.Profile{
testfixture.WithProfiles(testfixture.Profile{
ID: "profile", BackendID: "custom", Model: "model",
}),
)
@@ -766,41 +740,31 @@ func TestWithBackendCopiesQueueCapacity(t *testing.T) {
}
}
func TestBackendRegistrationRejectsInvalidAndDuplicateDefinitions(t *testing.T) {
func TestBackendConfigRejectsInvalidDefinitions(t *testing.T) {
cycle := map[string]any{}
cycle["self"] = cycle
tests := []struct {
name string
backends []promptkit.Backend
backends []testfixture.Backend
}{
{name: "blank id", backends: []promptkit.Backend{{Endpoint: "http://example.test/v1"}}},
{name: "invalid endpoint", backends: []promptkit.Backend{{ID: "custom", Endpoint: "ftp://example.test/v1"}}},
{name: "invalid environment", backends: []promptkit.Backend{{ID: "custom", Endpoint: "http://example.test/v1", APIKeyEnv: "BAD-NAME"}}},
{name: "reserved extra parameter", backends: []promptkit.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: map[string]any{"model": "override"}}}},
{name: "cyclic extra parameter", backends: []promptkit.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: cycle}}},
{name: "malformed JSON number", backends: []promptkit.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: map[string]any{"value": json.Number("01")}}}},
{name: "excessively deep extra parameter", backends: []promptkit.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: map[string]any{"value": excessivelyDeepJSONValue()}}}},
{name: "duplicate consumer id", backends: []promptkit.Backend{
{ID: " custom ", Endpoint: "http://one.example/v1"},
{ID: "custom", Endpoint: "http://two.example/v1"},
}},
{name: "reserved OpenRouter ID", backends: []promptkit.Backend{{
ID: promptkit.BackendOpenRouter, Endpoint: "http://replacement.example/v1",
}}},
{name: "reserved Rakestrawhome ID", backends: []promptkit.Backend{{
ID: promptkit.BackendRakestrawHome, Endpoint: "http://replacement.example/v1",
}}},
{name: "blank id", backends: []testfixture.Backend{{Endpoint: "http://example.test/v1"}}},
{name: "invalid endpoint", backends: []testfixture.Backend{{ID: "custom", Endpoint: "ftp://example.test/v1"}}},
{name: "invalid environment", backends: []testfixture.Backend{{ID: "custom", Endpoint: "http://example.test/v1", APIKeyEnv: "BAD-NAME"}}},
{name: "reserved extra parameter", backends: []testfixture.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: map[string]any{"model": "override"}}}},
{name: "cyclic extra parameter", backends: []testfixture.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: cycle}}},
{name: "malformed JSON number", backends: []testfixture.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: map[string]any{"value": json.Number("01")}}}},
{name: "excessively deep extra parameter", backends: []testfixture.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: map[string]any{"value": excessivelyDeepJSONValue()}}}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
options := []promptkit.Option{
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
options := []any{
testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
}
for _, backend := range tt.backends {
options = append(options, promptkit.WithBackend(backend))
options = append(options, testfixture.WithBackend(backend))
}
_, err := promptkit.NewEngine(promptkit.Config{}, options...)
_, err := testfixture.NewEngine(testfixture.Config{}, options...)
if !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("expected ErrInvalidConfig, got %v", err)
}
@@ -811,12 +775,12 @@ func TestBackendRegistrationRejectsInvalidAndDuplicateDefinitions(t *testing.T)
func TestBackendExtraParamsAreDeeplyCopiedAtConstructionAndLookup(t *testing.T) {
nested := map[string]any{"value": "original"}
extraParams := map[string]any{"nested": nested}
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
promptkit.WithBackend(promptkit.Backend{
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
testfixture.WithBackend(testfixture.Backend{
ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: extraParams,
}),
promptkit.WithProfiles(promptkit.Profile{ID: "profile", BackendID: "custom", Model: "model"}),
testfixture.WithProfiles(testfixture.Profile{ID: "profile", BackendID: "custom", Model: "model"}),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
@@ -956,13 +920,13 @@ func TestEngineRunReturnsFinalResultAfterRepairExhaustion(t *testing.T) {
}
func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) {
profile := promptkit.Profile{ID: "profile", Endpoint: "http://example.test/v1", Model: "model"}
profile := testfixture.Profile{ID: "profile", Endpoint: "http://example.test/v1", Model: "model"}
t.Run("prompt source", func(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("first", "profile", "first"), "."),
promptkit.WithPromptFS(contractPromptFS("second", "profile", "second"), "."),
promptkit.WithProfiles(profile),
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("first", "profile", "first"), "."),
testfixture.WithPromptFS(contractPromptFS("second", "profile", "second"), "."),
testfixture.WithProfiles(profile),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
@@ -977,10 +941,10 @@ func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) {
})
t.Run("profile source", func(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
promptkit.WithProfileFS(contractProfileFS("profile", "first-model"), "."),
promptkit.WithProfileFS(contractProfileFS("profile", "second-model"), "."),
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
testfixture.WithProfileFS(contractProfileFS("profile", "first-model"), "."),
testfixture.WithProfileFS(contractProfileFS("profile", "second-model"), "."),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
@@ -995,10 +959,10 @@ func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) {
})
t.Run("fallback profile source", func(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
promptkit.WithFallbackProfileFS(contractProfileFS("profile", "first-model"), "."),
promptkit.WithFallbackProfileFS(contractProfileFS("profile", "second-model"), "."),
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
testfixture.WithFallbackProfileFS(contractProfileFS("profile", "first-model"), "."),
testfixture.WithFallbackProfileFS(contractProfileFS("profile", "second-model"), "."),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
@@ -1017,10 +981,10 @@ func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) {
first.Model = "first-model"
second := profile
second.Model = "second-model"
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
promptkit.WithProfiles(first),
promptkit.WithProfiles(second),
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
testfixture.WithProfiles(first),
testfixture.WithProfiles(second),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
@@ -1035,9 +999,9 @@ func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) {
})
t.Run("schema source", func(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractSchemaPromptFS(), "."),
promptkit.WithProfiles(profile),
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractSchemaPromptFS(), "."),
testfixture.WithProfiles(profile),
promptkit.WithSchemaFS(contractSchemaFS("first"), "."),
promptkit.WithSchemaFS(contractSchemaFS("second"), "."),
)
@@ -1056,9 +1020,9 @@ func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) {
t.Run("model client", func(t *testing.T) {
var firstCalls, secondCalls atomic.Int64
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
promptkit.WithProfiles(profile),
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
testfixture.WithProfiles(profile),
promptkit.WithLLMClient(countingLLMClient{calls: &firstCalls}),
promptkit.WithLLMClient(countingLLMClient{calls: &secondCalls}),
)
@@ -1074,9 +1038,9 @@ func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) {
})
t.Run("artifact reader", func(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractInputPromptFS(), "."),
promptkit.WithProfiles(profile),
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractInputPromptFS(), "."),
testfixture.WithProfiles(profile),
promptkit.WithArtifactReader(fixedArtifactReader("first")),
promptkit.WithArtifactReader(fixedArtifactReader("second")),
)
@@ -1109,11 +1073,11 @@ func TestFallbackProfileSourcePrecedence(t *testing.T) {
}
t.Run("in-memory profiles override ordinary and fallback profiles", func(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
promptkit.WithProfiles(promptkit.Profile{ID: profileID, Endpoint: "http://example.test/v1", Model: "memory-model"}),
promptkit.WithProfileFS(contractProfileFS(profileID, "ordinary-model"), "."),
promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
testfixture.WithProfiles(testfixture.Profile{ID: profileID, Endpoint: "http://example.test/v1", Model: "memory-model"}),
testfixture.WithProfileFS(contractProfileFS(profileID, "ordinary-model"), "."),
testfixture.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
@@ -1124,10 +1088,10 @@ func TestFallbackProfileSourcePrecedence(t *testing.T) {
})
t.Run("ordinary filesystem source overrides fallback profile", func(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
promptkit.WithProfileFS(contractProfileFS(profileID, "ordinary-model"), "."),
promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
testfixture.WithProfileFS(contractProfileFS(profileID, "ordinary-model"), "."),
testfixture.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
@@ -1140,9 +1104,9 @@ func TestFallbackProfileSourcePrecedence(t *testing.T) {
t.Run("ordinary option replaces configured directory", func(t *testing.T) {
profileDir := t.TempDir()
writePublicProfileFile(t, profileDir, profileID, "http://example.test/v1", "directory-model")
engine, err := promptkit.NewEngine(promptkit.Config{ProfileDir: profileDir},
promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
promptkit.WithProfileFS(contractProfileFS(profileID, "option-model"), "."),
engine, err := testfixture.NewEngine(testfixture.Config{ProfileDir: profileDir},
testfixture.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
testfixture.WithProfileFS(contractProfileFS(profileID, "option-model"), "."),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
@@ -1155,9 +1119,9 @@ func TestFallbackProfileSourcePrecedence(t *testing.T) {
t.Run("configured directory overrides fallback profile", func(t *testing.T) {
profileDir := t.TempDir()
writePublicProfileFile(t, profileDir, profileID, "http://example.test/v1", "directory-model")
engine, err := promptkit.NewEngine(promptkit.Config{ProfileDir: profileDir},
promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
engine, err := testfixture.NewEngine(testfixture.Config{ProfileDir: profileDir},
testfixture.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
testfixture.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
@@ -1168,9 +1132,9 @@ func TestFallbackProfileSourcePrecedence(t *testing.T) {
})
t.Run("fallback profile overrides built-in profile", func(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."),
promptkit.WithFallbackProfileFS(contractProfileFS("mistral-small-3", "fallback-model"), "."),
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."),
testfixture.WithFallbackProfileFS(contractProfileFS("mistral-small-3", "fallback-model"), "."),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
@@ -1182,17 +1146,17 @@ func TestFallbackProfileSourcePrecedence(t *testing.T) {
t.Run("missing fallback profile uses built-in profile", func(t *testing.T) {
t.Setenv("OPENROUTER_API_KEY", "test-key")
baseline, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."),
baseline, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."),
)
if err != nil {
t.Fatalf("construct baseline engine: %v", err)
}
want := prepareModel(t, baseline, "prompt")
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."),
promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."),
testfixture.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
@@ -1210,8 +1174,8 @@ func TestPromptkitOwnedProfilesUseExistingProfileWorkflows(t *testing.T) {
)
t.Run("inspection and preparation resolve both names", func(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", standardID, "message"), "."),
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", standardID, "message"), "."),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
@@ -1241,9 +1205,9 @@ func TestPromptkitOwnedProfilesUseExistingProfileWorkflows(t *testing.T) {
})
t.Run("consumer definitions override an owned profile and its base", func(t *testing.T) {
auto, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("auto", autoID, "message"), "."),
promptkit.WithProfiles(promptkit.Profile{
auto, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("auto", autoID, "message"), "."),
testfixture.WithProfiles(testfixture.Profile{
ID: autoID,
Endpoint: "http://consumer.example/v1",
Model: "consumer-auto-model",
@@ -1260,9 +1224,9 @@ func TestPromptkitOwnedProfilesUseExistingProfileWorkflows(t *testing.T) {
t.Fatalf("owned profile override model = %q", autoPrepared.EffectiveModelParams.Model)
}
standard, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("standard", standardID, "message"), "."),
promptkit.WithProfiles(promptkit.Profile{
standard, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("standard", standardID, "message"), "."),
testfixture.WithProfiles(testfixture.Profile{
ID: "gemini-flash-latest",
Endpoint: "http://consumer.example/v1",
Model: "consumer-base-model",
@@ -1281,9 +1245,9 @@ func TestPromptkitOwnedProfilesUseExistingProfileWorkflows(t *testing.T) {
})
t.Run("prompt default selection remains explicit", func(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("default", "application-default", "message"), "."),
promptkit.WithProfiles(promptkit.Profile{
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("default", "application-default", "message"), "."),
testfixture.WithProfiles(testfixture.Profile{
ID: "application-default",
Endpoint: "http://consumer.example/v1",
Model: "consumer-default-model",
@@ -1302,69 +1266,56 @@ func TestPromptkitOwnedProfilesUseExistingProfileWorkflows(t *testing.T) {
})
}
func TestFallbackProfileSourcePreservesLazyLoadingAndErrors(t *testing.T) {
func TestInvalidFallbackProfileSourcesFailConstruction(t *testing.T) {
const profileID = "application-profile"
t.Run("construction defers malformed fallback profiles", func(t *testing.T) {
_, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(fstest.MapFS{}, "."),
promptkit.WithFallbackProfileFS(fstest.MapFS{
t.Run("malformed fallback profile", func(t *testing.T) {
_, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(fstest.MapFS{}, "."),
testfixture.WithFallbackProfileFS(fstest.MapFS{
"broken.yaml": &fstest.MapFile{Data: []byte("id: broken\nunknown: value\n")},
}, "."),
)
if err != nil {
t.Fatalf("construct engine with malformed fallback profile: %v", err)
if !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("expected invalid configuration, got %v", err)
}
})
t.Run("unrelated malformed fallback profile does not block matching definition", func(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
promptkit.WithFallbackProfileFS(fstest.MapFS{
t.Run("unrelated malformed fallback profile blocks construction", func(t *testing.T) {
_, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
testfixture.WithFallbackProfileFS(fstest.MapFS{
"broken.yaml": &fstest.MapFile{Data: []byte("id: unrelated\nunknown: value\n")},
"valid.yaml": &fstest.MapFile{Data: []byte("id: application-profile\nendpoint: http://example.test/v1\nmodel: fallback-model\n")},
}, "."),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatalf("prepare from valid fallback profile: %v", err)
}
if prepared.EffectiveModelParams.Model != "fallback-model" {
t.Fatalf("unexpected fallback profile model: %q", prepared.EffectiveModelParams.Model)
if !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("expected invalid configuration, got %v", err)
}
})
t.Run("matching malformed fallback profile does not reach built-in profile", func(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."),
promptkit.WithFallbackProfileFS(fstest.MapFS{
t.Run("matching malformed fallback profile", func(t *testing.T) {
_, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."),
testfixture.WithFallbackProfileFS(fstest.MapFS{
"mistral-small-3.yaml": &fstest.MapFile{Data: []byte("id: mistral-small-3\nendpoint: http://example.test/v1\nmodel: fallback-model\nunknown: value\n")},
}, "."),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
if _, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"}); !errors.Is(err, promptkit.ErrProfileLoad) {
t.Fatalf("expected ErrProfileLoad, got %v", err)
if !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("expected invalid configuration, got %v", err)
}
})
t.Run("matching malformed ordinary profile does not reach fallback profile", func(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
promptkit.WithProfileFS(fstest.MapFS{
t.Run("matching malformed ordinary profile", func(t *testing.T) {
_, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
testfixture.WithProfileFS(fstest.MapFS{
"application-profile.yaml": &fstest.MapFile{Data: []byte("id: application-profile\nendpoint: http://example.test/v1\nmodel: ordinary-model\nunknown: value\n")},
}, "."),
promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
testfixture.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
if _, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"}); !errors.Is(err, promptkit.ErrProfileLoad) {
t.Fatalf("expected ErrProfileLoad, got %v", err)
if !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("expected invalid configuration, got %v", err)
}
})
}
@@ -1372,9 +1323,9 @@ func TestFallbackProfileSourcePreservesLazyLoadingAndErrors(t *testing.T) {
func TestFallbackProfileSourceWorksAcrossWorkflows(t *testing.T) {
const profileID = "application-profile"
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
promptkit.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."),
testfixture.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."),
promptkit.WithLLMClient(client),
)
if err != nil {
@@ -1417,13 +1368,13 @@ func TestFallbackProfileSourceWorksAcrossWorkflows(t *testing.T) {
}
func TestEngineSupportsConcurrentPrepareAndRun(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
promptkit.WithBackend(promptkit.Backend{
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
testfixture.WithBackend(testfixture.Backend{
ID: "concurrent",
Endpoint: "http://example.test/v1",
}),
promptkit.WithProfiles(promptkit.Profile{
testfixture.WithProfiles(testfixture.Profile{
ID: "profile",
BackendID: "concurrent",
Model: "model",

25
reliable_execution_contract_test.go
View File

@@ -13,6 +13,7 @@ import (
"time"
"gitea.maximumdirect.net/eric/promptkit"
"gitea.maximumdirect.net/eric/promptkit/testfixture"
)
func observedHTTPResponse(status int, body string) *http.Response {
@@ -39,7 +40,7 @@ func TestPublicRetryRepairSequenceKeepsUsageAndEventsConsistent(t *testing.T) {
calls++
return r, nil
})}
engine, err := promptkit.NewEngine(config)
engine, err := testfixture.NewEngine(config)
if err != nil {
t.Fatal(err)
}
@@ -124,10 +125,10 @@ func TestProfilesSharingBackendCoordinateRetryRepairAndAdmission(t *testing.T) {
}
})
one := 1
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: transport}, TransientRetries: &one},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile-a", "hello"), "."),
promptkit.WithBackend(promptkit.Backend{ID: "limited", Endpoint: "https://example.invalid/v1", ConcurrencyLimit: 1, QueueCapacity: &one}),
promptkit.WithProfiles(promptkit.Profile{ID: "profile-a", BackendID: "limited", Model: "model-a"}, promptkit.Profile{ID: "profile-b", BackendID: "limited", Model: "model-b"}))
engine, err := testfixture.NewEngine(testfixture.Config{HTTPClient: &http.Client{Transport: transport}, TransientRetries: &one},
testfixture.WithPromptFS(contractPromptFS("prompt", "profile-a", "hello"), "."),
testfixture.WithBackend(testfixture.Backend{ID: "limited", Endpoint: "https://example.invalid/v1", ConcurrencyLimit: 1, QueueCapacity: &one}),
testfixture.WithProfiles(testfixture.Profile{ID: "profile-a", BackendID: "limited", Model: "model-a"}, testfixture.Profile{ID: "profile-b", BackendID: "limited", Model: "model-b"}))
if err != nil {
t.Fatal(err)
}
@@ -186,11 +187,11 @@ func TestDifferentBackendPoolsCanDispatchTogether(t *testing.T) {
return observedHTTPResponse(200, `{"choices":[{"message":{"content":"ok"}}]}`), nil
})
zero := 0
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: transport}, TransientRetries: &zero},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile-a", "hello"), "."),
promptkit.WithBackend(promptkit.Backend{ID: "backend-a", Endpoint: "https://example.invalid/v1", ConcurrencyLimit: 1, QueueCapacity: &zero}),
promptkit.WithBackend(promptkit.Backend{ID: "backend-b", Endpoint: "https://example.invalid/v1", ConcurrencyLimit: 1, QueueCapacity: &zero}),
promptkit.WithProfiles(promptkit.Profile{ID: "profile-a", BackendID: "backend-a", Model: "model-a"}, promptkit.Profile{ID: "profile-b", BackendID: "backend-b", Model: "model-b"}))
engine, err := testfixture.NewEngine(testfixture.Config{HTTPClient: &http.Client{Transport: transport}, TransientRetries: &zero},
testfixture.WithPromptFS(contractPromptFS("prompt", "profile-a", "hello"), "."),
testfixture.WithBackend(testfixture.Backend{ID: "backend-a", Endpoint: "https://example.invalid/v1", ConcurrencyLimit: 1, QueueCapacity: &zero}),
testfixture.WithBackend(testfixture.Backend{ID: "backend-b", Endpoint: "https://example.invalid/v1", ConcurrencyLimit: 1, QueueCapacity: &zero}),
testfixture.WithProfiles(testfixture.Profile{ID: "profile-a", BackendID: "backend-a", Model: "model-a"}, testfixture.Profile{ID: "profile-b", BackendID: "backend-b", Model: "model-b"}))
if err != nil {
t.Fatal(err)
}
@@ -251,8 +252,8 @@ func TestPreparedRetriesFreezeSourceAndCredentialAtExecution(t *testing.T) {
return observedHTTPResponse(200, `{"choices":[{"message":{"content":"ok"}}]}`), nil
})
one := 1
engine, err := promptkit.NewEngine(promptkit.Config{HTTPClient: &http.Client{Transport: transport}, TransientRetries: &one},
promptkit.WithPromptFS(source, "."), promptkit.WithProfiles(promptkit.Profile{ID: "profile", Endpoint: "https://example.invalid/v1", Model: "model"}))
engine, err := testfixture.NewEngine(testfixture.Config{HTTPClient: &http.Client{Transport: transport}, TransientRetries: &one},
testfixture.WithPromptFS(source, "."), testfixture.WithProfiles(testfixture.Profile{ID: "profile", Endpoint: "https://example.invalid/v1", Model: "model"}))
if err != nil {
t.Fatal(err)
}

359
testfixture/engine.go Normal file
View File

@@ -0,0 +1,359 @@
// Package testfixture keeps existing runtime regression scenarios executable
// while their source setup is expressed through the new configuration boundary.
package testfixture
import (
"bytes"
"fmt"
"io"
"io/fs"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit"
"gopkg.in/yaml.v3"
)
// Config carries the existing regression fixtures' construction settings.
type Config struct {
PromptDir string
ProfileDir string
SchemaDir string
Timeout time.Duration
HTTPClient *http.Client
TransientRetries *int
}
// Profile describes an existing regression fixture's execution settings.
type Profile struct {
ID, BaseProfileID, BackendID, Endpoint, Model string
Temperature, TopP float64
MaxTokens, TimeoutSeconds int
ServiceTier, ReasoningEffort string
APIKeyRequired bool
ExtraParams map[string]any
}
// OpenAICompatibleProfileConfig describes a profile fixture.
type OpenAICompatibleProfileConfig = Profile
// OpenAICompatibleProfile produces a profile fixture.
func OpenAICompatibleProfile(config OpenAICompatibleProfileConfig) Profile {
profile := Profile(config)
if config.ExtraParams != nil {
profile.ExtraParams = make(map[string]any, len(config.ExtraParams))
for key, value := range config.ExtraParams {
profile.ExtraParams[key] = value
}
}
return profile
}
// Backend describes an existing regression fixture's backend settings.
type Backend struct {
ID, Endpoint, APIKeyEnv string
ConcurrencyLimit int
QueueCapacity *int
ExtraParams map[string]any
}
// LocalBackend returns a conventional local backend fixture.
func LocalBackend(endpoint string, concurrencyLimit int) Backend {
return Backend{ID: promptkit.BackendLocal, Endpoint: endpoint, ConcurrencyLimit: concurrencyLimit}
}
type option func(*choices) error
type choices struct {
promptFS, profileFS, fallbackFS fs.FS
promptRoot, profileRoot, fallbackRoot string
promptFile, profileFile string
profiles []Profile
profilesSet bool
backends []Backend
}
// WithPromptFS selects prompt fixture files.
func WithPromptFS(fsys fs.FS, root string) option {
return func(c *choices) error {
if err := validSourceFS(fsys, root); err != nil {
return err
}
c.promptFS, c.promptRoot, c.promptFile = fsys, root, ""
return nil
}
}
// WithPromptFile selects one prompt fixture file.
func WithPromptFile(name string) option {
return func(c *choices) error {
if strings.TrimSpace(name) == "" {
return fmt.Errorf("prompt file is required")
}
c.promptFS, c.promptFile = nil, name
return nil
}
}
// WithProfileFS selects profile fixture files.
func WithProfileFS(fsys fs.FS, root string) option {
return func(c *choices) error {
if err := validSourceFS(fsys, root); err != nil {
return err
}
c.profileFS, c.profileRoot, c.profileFile = fsys, root, ""
return nil
}
}
// WithProfileFile selects one profile fixture file.
func WithProfileFile(name string) option {
return func(c *choices) error {
if strings.TrimSpace(name) == "" {
return fmt.Errorf("profile file is required")
}
c.profileFS, c.profileFile = nil, name
return nil
}
}
// WithFallbackProfileFS selects lower-precedence profile fixture files.
func WithFallbackProfileFS(fsys fs.FS, root string) option {
return func(c *choices) error {
if err := validSourceFS(fsys, root); err != nil {
return err
}
c.fallbackFS, c.fallbackRoot = fsys, root
return nil
}
}
func validSourceFS(filesystem fs.FS, root string) error {
if filesystem == nil || strings.TrimSpace(root) == "" {
return fmt.Errorf("source filesystem and root are required")
}
return nil
}
// WithProfiles selects in-memory profile fixtures.
func WithProfiles(profiles ...Profile) option {
return func(c *choices) error { c.profiles, c.profilesSet = profiles, true; return nil }
}
// WithBackend adds one backend fixture.
func WithBackend(backend Backend) option {
if backend.QueueCapacity != nil {
capacity := *backend.QueueCapacity
backend.QueueCapacity = &capacity
}
return func(c *choices) error { c.backends = append(c.backends, backend); return nil }
}
// NewEngine expresses existing regression scenarios through ConfigKit sources.
func NewEngine(config Config, opts ...any) (*promptkit.Engine, error) {
var choices choices
var runtime []promptkit.Option
for _, candidate := range opts {
switch value := candidate.(type) {
case nil:
case option:
if err := value(&choices); err != nil {
return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err)
}
case promptkit.Option:
runtime = append(runtime, value)
default:
return nil, fmt.Errorf("%w: unsupported fixture option %T", promptkit.ErrInvalidConfig, candidate)
}
}
var underlays []configkit.Underlay
add := func(inputs []configkit.Input, err error) error {
if err != nil {
return err
}
if len(inputs) > 0 {
underlays = append(underlays, configkit.Underlay{Inputs: inputs})
}
return nil
}
if choices.fallbackFS != nil {
if err := add(sourceInputs("fallback profiles", choices.fallbackFS, choices.fallbackRoot, "profiles")); err != nil {
return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err)
}
}
if choices.profileFile != "" {
if err := add(singleInput(choices.profileFile, "profiles")); err != nil {
return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err)
}
} else if choices.profileFS != nil {
if err := add(sourceInputs("profiles", choices.profileFS, choices.profileRoot, "profiles")); err != nil {
return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err)
}
} else if config.ProfileDir != "" {
if err := add(directoryInputs(config.ProfileDir, "profiles")); err != nil {
return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err)
}
}
if choices.profilesSet {
profiles := make(map[string]promptkit.ProfileConfig, len(choices.profiles))
for _, value := range choices.profiles {
if _, exists := profiles[value.ID]; exists {
return nil, fmt.Errorf("%w: duplicate profile fixture %q", promptkit.ErrInvalidConfig, value.ID)
}
profiles[value.ID] = profileConfig(value)
}
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("memory profiles", profiles)}}})
}
for index, value := range choices.backends {
config := promptkit.BackendConfig{Endpoint: &value.Endpoint, APIKeyEnv: &value.APIKeyEnv,
ConcurrencyLimit: &value.ConcurrencyLimit, QueueCapacity: value.QueueCapacity, ExtraParams: value.ExtraParams}
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{{Source: promptkit.BackendSource(fmt.Sprintf("backend %d", index), map[string]promptkit.BackendConfig{value.ID: config})}}})
}
if choices.promptFile != "" {
if err := add(singleInput(choices.promptFile, "prompts")); err != nil {
return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err)
}
} else if choices.promptFS != nil {
if err := add(sourceInputs("prompts", choices.promptFS, choices.promptRoot, "prompts")); err != nil {
return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err)
}
} else if config.PromptDir != "" {
if err := add(directoryInputs(config.PromptDir, "prompts")); err != nil {
return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err)
}
}
return promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: underlays},
SchemaDir: config.SchemaDir, Timeout: config.Timeout, HTTPClient: config.HTTPClient,
TransientRetries: config.TransientRetries}, runtime...)
}
func profileConfig(value Profile) promptkit.ProfileConfig {
config := promptkit.ProfileConfig{ExtraParams: value.ExtraParams}
if value.BaseProfileID != "" {
config.Extends = &value.BaseProfileID
}
if value.BackendID != "" {
config.Backend = &value.BackendID
}
if value.Endpoint != "" {
config.Endpoint = &value.Endpoint
}
if value.Model != "" {
config.Model = &value.Model
}
if value.Temperature != 0 {
config.Temperature = &value.Temperature
}
if value.TopP != 0 {
config.TopP = &value.TopP
}
if value.MaxTokens != 0 {
config.MaxTokens = &value.MaxTokens
}
if value.TimeoutSeconds != 0 {
config.TimeoutSeconds = &value.TimeoutSeconds
}
if value.ServiceTier != "" {
config.ServiceTier = &value.ServiceTier
}
if value.ReasoningEffort != "" {
config.ReasoningEffort = &value.ReasoningEffort
}
if value.APIKeyRequired {
config.APIKeyRequired = &value.APIKeyRequired
}
return config
}
func directoryInputs(directory, collection string) ([]configkit.Input, error) {
if _, err := os.Stat(directory); err != nil {
return nil, err
}
return sourceInputs(directory, os.DirFS(directory), ".", collection)
}
func sourceInputs(label string, filesystem fs.FS, root, collection string) ([]configkit.Input, error) {
return promptkit.FilesystemInputs(label, convertingFS{FS: filesystem, collection: collection}, root)
}
func singleInput(name, collection string) ([]configkit.Input, error) {
info, err := os.Stat(name)
if err != nil {
return nil, err
}
if info.IsDir() {
return nil, fmt.Errorf("source %s is a directory", name)
}
filesystem := convertingFS{FS: os.DirFS(filepath.Dir(name)), collection: collection}
namespace := configkit.NewFilesystem(name, filesystem)
return []configkit.Input{{Source: namespace.File(filepath.Base(name)), Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: "."}}}, nil
}
type convertingFS struct {
fs.FS
collection string
}
func (source convertingFS) Open(name string) (fs.File, error) {
file, err := source.FS.Open(name)
if err != nil || (!strings.HasSuffix(strings.TrimSpace(name), ".yaml") && !strings.HasSuffix(strings.TrimSpace(name), ".yml")) {
return file, err
}
info, err := file.Stat()
if err != nil {
file.Close()
return nil, err
}
data, err := io.ReadAll(file)
closeErr := file.Close()
if err != nil {
return nil, err
}
if closeErr != nil {
return nil, closeErr
}
converted := convertDocument(data, source.collection)
return &memoryFile{Reader: bytes.NewReader(converted), info: info}, nil
}
type memoryFile struct {
*bytes.Reader
info fs.FileInfo
}
func (file *memoryFile) Stat() (fs.FileInfo, error) { return file.info, nil }
func (file *memoryFile) Close() error { return nil }
func convertDocument(data []byte, collection string) []byte {
var document map[string]any
if err := yaml.Unmarshal(data, &document); err != nil || document == nil {
return data
}
if _, keyed := document[collection]; keyed {
return data
}
id, idOK := document["id"]
if !idOK {
return data
}
delete(document, "id")
if collection == "profiles" {
if base, exists := document["base_profile"]; exists {
delete(document, "base_profile")
document["extends"] = base
}
data, _ = yaml.Marshal(map[string]any{"profiles": map[string]any{fmt.Sprint(id): document}})
return data
}
version, versionOK := document["version"]
if !versionOK {
return data
}
delete(document, "version")
data, _ = yaml.Marshal(map[string]any{"prompts": map[string]any{fmt.Sprint(id): map[string]any{fmt.Sprint(version): document}}})
return data
}

5
transient_retries_contract_test.go
View File

@@ -11,6 +11,7 @@ import (
"testing"
"gitea.maximumdirect.net/eric/promptkit"
"gitea.maximumdirect.net/eric/promptkit/testfixture"
)
func retryHTTPResponse(status int, content string) *http.Response {
@@ -30,7 +31,7 @@ func retryTestEngine(t *testing.T, budget *int, next func(int) *http.Response) (
calls++
return next(calls), nil
})}
engine, err := promptkit.NewEngine(config)
engine, err := testfixture.NewEngine(config)
if err != nil {
t.Fatal(err)
}
@@ -71,7 +72,7 @@ func TestRetryValidationAndPreparedFreezing(t *testing.T) {
negative := -1
config := contractConfig(frameworkSchemaDir)
config.TransientRetries = &negative
if _, err := promptkit.NewEngine(config); !errors.Is(err, promptkit.ErrInvalidConfig) {
if _, err := testfixture.NewEngine(config); !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("negative engine budget = %v", err)
}

105
types.go
View File

@@ -468,111 +468,6 @@ type ExecutionTargetOverride struct {
ExtraParams map[string]any
}
// Profile is an in-memory execution profile for library consumers.
//
// A standalone Profile is equivalent to a loaded profile file after local
// validation. A derived profile names BaseProfileID and can inherit target
// fields when selected or inspected. Raw API keys do not belong in profiles;
// use APIKeyRequired to require callers to provide a RunRequest.APIKey or
// explicit request ExecutionTargetOverride.APIKeyEnv, or use profile YAML
// api_key_env with file and FS profile sources. Profile has no stable JSON
// representation.
//
// WithProfiles locally validates and copies Profile values during NewEngine.
// It checks base-reference existence and resolved target completeness when a
// derived profile is selected or inspected. Zero Temperature, MaxTokens, and
// TopP values and blank ServiceTier and ReasoningEffort values leave those
// provider controls unspecified. A zero TimeoutSeconds retains the framework
// deadline, while an empty ExtraParams map inherits backend request defaults.
// Use ExecutionTargetOverride pointer fields to request an explicit numeric
// zero.
type Profile struct {
// ID is the required non-blank profile identifier. WithProfiles trims it.
ID string
// BaseProfileID optionally names one base profile. WithProfiles trims it. A
// non-blank value permits required target fields to be inherited when the
// profile is selected or inspected, which is also when reference existence
// and resolved completeness are checked. A blank value leaves this as a
// standalone profile.
BaseProfileID string
// BackendID optionally selects an engine backend. WithProfiles trims it.
// Backend membership is checked when a request selects the profile; an
// unknown ID makes preparation fail with ErrProfileLoad.
BackendID string
// Endpoint is the model-provider base URL. A standalone Profile requires an
// endpoint when BackendID is blank; a derived Profile may inherit either
// field. A non-blank endpoint overrides the backend endpoint when
// non-blank. WithProfiles trims it and requires an absolute HTTP or HTTPS URL
// with a host and no user information, query, or fragment.
Endpoint string
// Model is the provider model identifier. It is required for a standalone
// Profile and may be inherited by a derived Profile.
Model string
// Temperature is from 0 through 2. Zero leaves the provider control
// unspecified.
Temperature float64
// MaxTokens is non-negative. Zero leaves the provider control unspecified.
MaxTokens int
// TopP is from 0 through 1. Zero leaves the provider control unspecified
// rather than selecting an explicit zero.
TopP float64
// TimeoutSeconds is non-negative. Zero retains the framework deadline.
TimeoutSeconds int
// ServiceTier is optional; a blank value leaves it unspecified.
ServiceTier string
// ReasoningEffort is optional; a blank value leaves it unspecified.
ReasoningEffort string
// APIKeyRequired clears a backend's inherited API-key environment name and
// requires a non-blank RunRequest.APIKey unless the request explicitly
// supplies ExecutionTargetOverride.APIKeyEnv. When false, a named
// environment source remains optional. It does not store a credential.
APIKeyRequired bool
// ExtraParams contains provider-specific JSON-compatible values. An empty
// map inherits backend request defaults, when any. WithProfiles validates
// and deeply copies it during NewEngine. Excessively deep or large values
// are rejected for safety.
ExtraParams map[string]any
}
// OpenAICompatibleProfileConfig configures an OpenAI-compatible in-memory
// profile.
//
// It contains ordinary profile fields for OpenAI-compatible chat-completions
// endpoints. BaseProfileID and APIKeyRequired follow Profile. Raw API keys do
// not belong in this config. OpenAICompatibleProfileConfig has no stable JSON
// representation and is not validated until its resulting Profile is supplied
// through WithProfiles to NewEngine.
type OpenAICompatibleProfileConfig struct {
// ID becomes Profile.ID.
ID string
// BaseProfileID becomes Profile.BaseProfileID. A non-blank value permits the
// resulting Profile to inherit target fields when it is selected or inspected.
BaseProfileID string
// BackendID becomes Profile.BackendID.
BackendID string
// Endpoint becomes Profile.Endpoint.
Endpoint string
// Model becomes Profile.Model.
Model string
// APIKeyRequired becomes Profile.APIKeyRequired.
APIKeyRequired bool
// Temperature becomes Profile.Temperature.
Temperature float64
// MaxTokens becomes Profile.MaxTokens.
MaxTokens int
// TopP becomes Profile.TopP.
TopP float64
// TimeoutSeconds becomes Profile.TimeoutSeconds.
TimeoutSeconds int
// ServiceTier becomes Profile.ServiceTier.
ServiceTier string
// ReasoningEffort becomes Profile.ReasoningEffort.
ReasoningEffort string
// ExtraParams becomes a shallow-copied Profile.ExtraParams map. NewEngine
// performs validation and a deep copy when WithProfiles applies the result.
ExtraParams map[string]any
}
// ExecutionTargetPresence reports which effective numeric settings should be
// sent even when zero. It has a stable JSON representation and lets injected
// clients preserve omission semantics.