eric/promptkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove superseded configuration loaders and consolidate runtime tests

Browse Source
This commit is contained in:
Eric Rakestraw
2026-09-30 04:18:27 +00:00
parent f17a38f60b
commit 48343300e2
98 changed files with 817 additions and 12180 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
appended_messages_contract_test.go
View File

@@ -8,7 +8,6 @@ import (
"testing"
"gitea.maximumdirect.net/eric/promptkit"
"gitea.maximumdirect.net/eric/promptkit/testfixture"
)
func TestAppendedMessageRoleConstantsAreStrings(t *testing.T) {
@@ -30,15 +29,7 @@ func TestAppendedMessagesComposeFrozenEffectivePrompt(t *testing.T) {
{Role: promptkit.RoleUser, Content: "corrective request", CacheControl: cacheControl},
}
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "output"}}
engine, err := testfixture.NewEngine(
testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "configured prompt"), "."),
testfixture.WithProfiles(testfixture.Profile{ID: "profile", Endpoint: "http://example.test/v1", Model: "model"}),
promptkit.WithLLMClient(client),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
engine := newSimpleEngine(t, "configured prompt", promptkit.WithLLMClient(client))
request := promptkit.RunRequest{PromptID: "prompt", AppendedMessages: appended}
plain, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})

52
capacity_contract_test.go
View File

@@ -7,8 +7,8 @@ import (
"testing"
"time"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit"
"gitea.maximumdirect.net/eric/promptkit/testfixture"
)
func TestEngineLimitsInjectedClientConcurrency(t *testing.T) {
@@ -151,17 +151,7 @@ func TestUnlimitedBackendsRetainInjectedClientConcurrency(t *testing.T) {
{
name: "endpoint-only profile",
configure: func(t *testing.T, client promptkit.LLMClient) *promptkit.Engine {
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
testfixture.WithProfiles(testfixture.Profile{
ID: "profile", Endpoint: "http://endpoint.example/v1", Model: "model",
}),
promptkit.WithLLMClient(client),
)
if err != nil {
t.Fatalf("construct endpoint-only engine: %v", err)
}
return engine
return newSimpleEngine(t, "message", promptkit.WithLLMClient(client))
},
},
}
@@ -251,27 +241,35 @@ func newBackendCapacityEngine(
reader promptkit.ArtifactReader,
) *promptkit.Engine {
t.Helper()
promptFS := contractPromptFS("prompt", "profile", "message")
promptID, content := "prompt", "message"
var inputs []any
if reader != nil {
promptFS = contractInputPromptFS()
promptID, content = "input-prompt", `{{input "input"}}`
inputs = []any{map[string]any{"name": "input", "required": true}}
}
options := []any{
testfixture.WithPromptFS(promptFS, "."),
testfixture.WithBackend(testfixture.Backend{
ID: "limited",
Endpoint: "http://backend.example/v1",
ConcurrencyLimit: limit,
QueueCapacity: queueCapacity,
}),
testfixture.WithProfiles(testfixture.Profile{
ID: "profile", BackendID: "limited", Model: "model",
}),
promptkit.WithLLMClient(client),
backend := map[string]any{"endpoint": "http://backend.example/v1", "concurrency_limit": limit}
if queueCapacity != nil {
backend["queue_capacity"] = *queueCapacity
}
definition := map[string]any{
"default_profile": "profile",
"messages": []any{map[string]any{"role": "user", "content": content}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
}
if inputs != nil {
definition["inputs"] = inputs
}
options := []promptkit.Option{promptkit.WithLLMClient(client)}
if reader != nil {
options = append(options, promptkit.WithArtifactReader(reader))
}
engine, err := testfixture.NewEngine(testfixture.Config{}, options...)
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("capacity fixture", map[string]any{
"backends": map[string]any{"limited": backend},
"profiles": map[string]any{"profile": map[string]any{"backend": "limited", "model": "model"}},
"prompts": map[string]any{promptID: map[string]any{"1": definition}},
}),
}}}, options...)
if err != nil {
t.Fatalf("construct capacity engine: %v", err)
}

73
consumer_test_helpers_test.go Normal file
View File

@@ -0,0 +1,73 @@
package promptkit_test
import (
"context"
"fmt"
"net/http"
"sync/atomic"
"testing"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit"
)
type fakeLLMClient struct {
response *promptkit.GenerateResponse
responses []*promptkit.GenerateResponse
err error
requests []promptkit.GenerateRequest
}
func (f *fakeLLMClient) Generate(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
f.requests = append(f.requests, request)
if f.err != nil {
return nil, f.err
}
if len(f.responses) > 0 {
index := len(f.requests) - 1
if index >= len(f.responses) {
return nil, fmt.Errorf("no response configured for generation %d", index+1)
}
return f.responses[index], nil
}
return f.response, nil
}
type countingLLMClient struct{ calls *atomic.Int64 }
func (client countingLLMClient) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
if client.calls != nil {
client.calls.Add(1)
}
return &promptkit.GenerateResponse{Content: "ok"}, nil
}
type roundTripFunc func(*http.Request) (*http.Response, error)
func (f roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) { return f(request) }
// newSimpleEngine exercises the public constructor with a complete keyed
// document. Tests that need layering or resources select their own inputs.
func newSimpleEngine(t *testing.T, content string, options ...promptkit.Option) *promptkit.Engine {
t.Helper()
engine, err := promptkit.NewEngine(simpleConfig(content), options...)
if err != nil {
t.Fatalf("construct consumer test engine: %v", err)
}
return engine
}
func simpleConfig(content string) promptkit.Config {
return promptkit.Config{Sources: promptkit.ConfigurationSources{
Main: &configkit.Input{Source: configkit.Mapping("consumer test", map[string]any{
"profiles": map[string]any{"profile": map[string]any{
"endpoint": "http://example.test/v1", "model": "model",
}},
"prompts": map[string]any{"prompt": map[string]any{"1": map[string]any{
"default_profile": "profile",
"messages": []any{map[string]any{"role": "user", "content": content}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
}}},
})},
}}
}

19
docs/internal/overview.md
View File

@@ -11,29 +11,28 @@ contributor workflow and validation.
| Component | Implemented responsibility | References |
| --- | --- | --- |
| Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request, result, prompt-inspection, profile-inspection, and ordinary-run observation values, opaque prepared-execution handles, profile construction, extension interfaces, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go) |
| `examples/go-library/prepare` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, and `Prepare`. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
| Root `promptkit` package | Provides the supported engine facade, ConfigKit source helpers, profile and backend mapping conveniences, injection options, public requests and results, explicit inspection, observation values, opaque prepared-execution handles, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go) |
| `examples/go-library/prepare` | Demonstrates offline preparation and explicit configuration inspection with a prompt file, programmatic profile, and inline input. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) |
| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, programmatic profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) |
| `examples/go-library/observe` | Demonstrates offline built-in HTTP retry and synchronous metadata observation across structured proposal and assessment runs using embedded sources, two profiles on one finite backend, and an application decision. It is not a public library package. | [Example program](../../examples/go-library/observe/main.go) |
| `internal/backend` | Constructs each engine's immutable registry from maintained definitions and consumer additions, validates and defensively copies definitions through the shared JSON-value package, and consumes the LLM-owned OpenAI-compatible reserved request-field rule. | [Backend registry](../../internal/backend/registry.go) |
| `internal/backend` | Holds a defensive, immutable lookup and capacity-policy index for final backend definitions validated by the configuration adapter. | [Backend registry](../../internal/backend/registry.go) |
| `internal/catalog` | Selects released keyed catalog assets and PromptKit-owned profile definitions as ConfigKit underlays; effective validation belongs to the configuration loader. | [Catalog adapter](../../internal/catalog/catalog.go), [internal sources](sources.md#profiles-and-built-ins) |
| `internal/configuration` | Loads keyed backend, profile, and prompt configuration through ConfigKit; records effective field presence, validates final values, captures indexed prompt definitions and referenced message content, and constructs immutable backend and presence-aware profile snapshots. It is not yet connected to engine assembly. | [Typed loader](../../internal/configuration/load.go), [internal sources](sources.md#typed-configuration-loading) |
| `internal/configuration` | Loads keyed backend, profile, and prompt configuration through ConfigKit; records effective field presence, validates final values, captures indexed prompt definitions and referenced message content, constructs backend and presence-aware profile snapshots, and retains an explicit read-only inspection view. | [Typed loader](../../internal/configuration/load.go), [internal sources](sources.md#typed-configuration-loading) |
| `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) |
| `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, and output contracts. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [endpoint invariant](../../internal/domain/endpoint.go) |
| `internal/engine` | Assembles external catalogs, Promptkit-owned profiles, repositories, registry, capacity, collaborators, repairer, and the one use-case runner for each engine instance. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#profiles-and-built-ins), [internal runner](runner.md) |
| `internal/engine` | Assembles selected catalog and caller configuration into one snapshot, then constructs capacity, rendering, validation, generation, repair, and the use-case runner for each engine. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#profiles-and-built-ins), [internal runner](runner.md) |
| `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, and attempt observations. | [Internal generation coordination](generation.md) |
| `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) |
| `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) |
| `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types and rejecting cycles or excessive depth and work. | [JSON values](../../internal/jsonvalue/jsonvalue.go) |
| `internal/legacycatalog` | Temporarily supports the existing public engine construction path with the previously published flat catalogs until engine assembly switches to ConfigKit. | [Existing catalog adapter](../../internal/legacycatalog/catalog.go), [internal sources](sources.md#profiles-and-built-ins) |
| `internal/promptdef` | Loads strictly decoded, validated prompt definitions from filesystem and `fs.FS` sources, including version selection and contained file-backed message content. | [Framework formats](../formats.md), [prompt-definition repository](../../internal/promptdef/filesystem_repository.go) |
| `internal/profile` | Owns immutable in-memory and filesystem profile repositories, local profile validation, raw-source overlays with error-preserving fallback, and inherited-profile resolution. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go), [internal sources](sources.md#profiles-and-built-ins) |
| `internal/promptdef` | Provides a narrow captured-prompt lookup contract, not-found identity, and execution-ready definition hash. | [Prompt lookup](../../internal/promptdef/repository.go), [framework formats](../formats.md) |
| `internal/profile` | Provides a narrow captured-profile lookup contract and not-found identity. | [Profile lookup](../../internal/profile/repository.go), [framework formats](../formats.md) |
| `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) |
| `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) |
| `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates operation-local validation plans with canonical contained schema resources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) |
| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded structured non-success response decoding, successful-response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) |
| `internal/observation` | Owns invocation-local run IDs, presence-aware usage and phase accounting, and synchronous metadata event delivery for ordinary and claimed prepared runs and generation attempts. | [Internal execution observation](observation.md) |
| `internal/usecase` | Resolves prompt definitions and hashes, profiles, backends, and targets for exact inspection and request settings for preparation, and coordinates ordinary and single-claim prepared execution across internal sources, rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
| `internal/usecase` | Selects captured prompts and profiles, applies request overrides and completion checks, and coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) |
The internal engine assembles these internal components without exposing their
representations. Consumers depend only on the root facade.

61
docs/internal/runner.md
View File

@@ -4,7 +4,7 @@
This document describes Promptkit's implemented internal orchestration. The
[architecture policy](../policy/architecture.md) owns dependency and consumer
boundaries. The [source and validation document](sources.md) owns repository,
boundaries. The [source and validation document](sources.md) owns configuration,
artifact, rendering, and validation behavior, while the
[model-client document](llm.md) owns generation behavior and failure
categories.
@@ -16,11 +16,10 @@ and override semantics consumed by the runner.
## Collaborators
`Runner` coordinates narrow internal interfaces for prompt definitions,
profiles, backend resolution, artifacts, rendering, model generation, and
validation. The internal engine supplies one immutable registry containing the
built-in backend and validated consumer additions, one engine-local run
admitter, and a generation coordinator using the same capacity manager. Validation
`Runner` coordinates narrow internal interfaces for captured prompt definitions
and resolved profiles, artifacts, rendering, model generation, and validation.
The internal engine supplies one engine-local run admitter and a generation
coordinator using the same capacity manager. Validation
plans and provider-facing schema metadata come from the validator's preparation
interface.
The internal engine supplies one default output repairer through the explicit
@@ -35,11 +34,11 @@ The runner has no durable run or session store.
The runner uses one prompt-selection and hashing boundary for ordinary
preparation and exact prompt inspection. Preparation retains its early
request-ID check before direct-session normalization; both operations then use
the configured prompt repository to select one definition, load referenced
message content, and calculate the same prompt hash.
request-ID check before direct-session normalization; both operations select
one captured definition and its construction-time hash. Referenced message
content and static template syntax were captured when the engine was built.
Inspection stops after that structural lookup. It does not parse templates or
Inspection stops after that structural lookup. It does not render templates or
touch profile, artifact, schema, renderer, validator, admission, or model
collaborators. The root [`Engine.InspectPrompt`](../../engine.go) GoDoc owns
the public operation's exact contract.
@@ -49,20 +48,11 @@ the public operation's exact contract.
The runner uses one profile-selection and target-resolution boundary for
ordinary preparation and exact profile inspection. Preparation first selects a
request profile or a prompt default; inspection begins with its required
explicit profile ID. Both then apply the ordinary source precedence, resolve a
named backend, and construct the effective target from framework, backend, and
profile values.
The private ConfigKit loader can instead supply an already resolved,
presence-aware target to this boundary. Request overrides still apply afterward,
and preparation checks endpoint and model completeness only after those
overrides. This path remains internal until the engine assembly switches to the
loader; the existing public engine still uses the repository path above.
For the existing repository path, inspection validates the resulting endpoint
and model. For the precomputed target path it can report an incomplete profile;
preparation remains responsible for checking completeness after request
overrides. Neither path checks credential availability or performs prompt,
explicit profile ID. Both use the captured, presence-aware target constructed
from the final effective profile and backend configuration. Request overrides
apply afterward; preparation checks endpoint and model completeness only after
those overrides. Inspection can report a partial profile without requiring a
runnable target. Neither path checks credential availability or performs prompt,
artifact, schema, rendering, admission, or model-client work. The root
[`Engine.InspectProfile`](../../engine.go) GoDoc owns the public operation's
exact contract.
@@ -74,11 +64,10 @@ where a run can be assigned to its selected backend pool. The resolution phase
performs only the work needed to validate routing and admission:
1. validate the required prompt selection and normalize any direct session ID;
2. load the prompt definition and hash the original definition;
2. select the captured prompt definition and hash;
3. select the request profile or the prompt's default profile;
4. resolve the profile's backend ID, when present;
5. resolve application-neutral defaults, backend defaults, profile values,
and explicit request overrides in that order;
4. retain the profile's selected backend identity, when present;
5. apply explicit request overrides to its captured backend-completed target;
6. validate endpoint, model, numeric overrides, and credential requirements;
7. resolve and validate the effective output contract without loading its
schema; and
@@ -110,26 +99,26 @@ Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
out-of-range values fail as invalid requests. Endpoint overrides do not change
the selected backend identity. Non-empty extra-parameter maps replace whole
lower-precedence maps. A direct API key takes precedence over environment
lookup; otherwise request, profile, and backend environment-variable names
apply in that order. A profile requiring a direct key clears an inherited
lookup; otherwise request, explicit profile, and backend environment-variable
names apply in that order. A profile requiring a direct key suppresses an inherited
backend environment name unless the request supplies its own name. Secret
values remain excluded from serialized metadata.
Reasoning overrides are tri-state: nil inherits the profile, a pointer to a
nonblank string trims and replaces it, and a pointer to a blank string clears
it. A nonblank direct session is normalized before source loading, bypasses
it. A nonblank direct session is normalized before prompt selection, bypasses
the prompt session template, and is applied after ordinary message rendering.
A blank direct value retains prompt-template behavior. The runner clears the
template only on a value copy of the definition, so the definition hash always
describes the original source while the rendered-prompt hash includes the
describes captured effective content while the rendered-prompt hash includes the
effective direct or rendered session and the complete effective message
sequence. The rendered-prompt hash uses a versioned, length-framed SHA-256
encoding of that session, every role and content value, and cache-control
presence and values; its hexadecimal value is opaque.
The registry is read-only after engine construction. Concurrent `Prepare` and
`Run` calls resolve independent defensive backend values and keep all
invocation state local.
The configuration snapshot is read-only after engine construction. Concurrent
`Prepare` and `Run` calls copy their resolved targets and keep all invocation
state local.
## Run Flow
@@ -223,7 +212,7 @@ generation, validation, repair, and cancellation failures.
Other context cancellation propagates through the invoked collaborator and is
classified by the owning operation. In particular, cancellation observed by
validation retains the context identity through the validation error category.
An overlong direct session is an invalid request before source loading, while
An overlong direct session is an invalid request before prompt selection, while
an invalid or overlong prompt session template remains a prompt-render failure.
An unknown selected backend, or a selected backend with no configured resolver,
is classified as a profile-load failure.

183
docs/internal/sources.md
View File

@@ -2,141 +2,68 @@
## Purpose
This document describes Promptkit's implemented internal source, artifact,
This document describes Promptkit's implemented configuration, artifact,
rendering, and output-validation behavior. The
[architecture policy](../policy/architecture.md) owns the library boundary and
dependency rules. None of these internal packages is a supported consumer API,
and the internal engine assembles them behind the public facade's source options and values.
The [framework format reference](../formats.md) owns the exact file fields,
validation modes, built-in catalog, and source precedence.
[architecture policy](../policy/architecture.md) owns dependency rules, while
the [framework format reference](../formats.md) owns fields, defaults, and
source semantics. Internal packages are not consumer APIs.
## Typed Configuration Loading
`internal/configuration` now contains a private ConfigKit adapter for keyed
backend, profile, and prompt mappings. It performs one layered load, records
effective field presence (including explicit null), normalizes backend and
profile fields through ConfigKit resolution, and validates final backend and
profile domain values. A lower-layer domain value replaced before the handoff
is not separately validated. It also validates effective prompt fields, reads
referenced message content using the contributing field's passive source
context, checks static template syntax, and indexes immutable definitions by
exact ID and version. The loader also builds an immutable backend registry from
the final definitions and indexes profile execution targets after inheritance,
applying the selected backend's defaults only then. These targets retain
explicit numeric-zero presence and null suppression for runtime execution.
Its focused tests exercise real ConfigKit loading.
`internal/configuration` is the single adapter from ConfigKit's final
effective configuration to execution-owned definitions. It performs one
layered load, captures effective presence (including explicit null), records
normalization and defaults through resolution, and validates all effective
backend, profile, and prompt values. Replaced lower-layer domain values are
not separately validated. It builds an immutable backend index and
presence-aware profile targets, applying final backend defaults only after
profile inheritance. It retains ConfigKit's result privately for explicit
inspection, not for per-run resolution. The runtime/domain packages do not
depend on ConfigKit.
This loader is not yet wired into engine assembly. The public engine continues
to use the source and repository behavior described below. The public
`DirectoryInputs` and `FilesystemInputs` helpers select sorted YAML inputs for
one strict-composition group and attach an explicit resource root; they do not
assign precedence or enable imports.
The root `DirectoryInputs` and `FilesystemInputs` helpers discover sorted YAML
files under an explicit root, provide resource context, and leave imports
disabled. Each returned slice belongs to one strict-composition group; file
order does not confer priority. The engine prepends selected catalog and
Promptkit-owned definitions, then caller underlays and optional main. The
[configuration tests](../../internal/configuration/load_test.go) exercise the
real loader and final-value handoff.
## Prompt Definitions
`internal/promptdef` uses one source-neutral flow for prompt selection and
normalization. That flow scans normalized YAML ID and version metadata,
requires one strictly decoded document per file, classifies errors for the
selected definition, detects duplicates, and normalizes the exact match.
Small operating-system and `fs.FS` adapters own discovery, byte reads, display
paths, content opening, and root containment. Each lookup remains a
point-in-time scan: definitions and catalogs are not cached, and file-backed
message content is opened only for the exact selected candidate.
The configuration adapter indexes each prompt by exact ID and version,
materializes file-backed message content using the exact effective field's
origin and resource context, and checks static message/session template syntax
at construction. It stores captured content, compiled templates, declared
input metadata, and a definition hash. It closes content-file handles before
returning and retains no source capability for public inspection.
Message roles are normalized through the shared domain owner by trimming
Unicode whitespace and lowercasing. Only `developer`, `system`, `user`, and
`assistant` are published; invalid roles remain selected prompt-definition
failures rather than becoming request errors. Cache-control metadata uses the
same shared domain normalization and defensive-copy rule.
Operating-system sources enforce containment against canonical roots and
targets so symlinks cannot escape. Injected `fs.FS` sources enforce containment
in their clean relative path namespace. A single-file source uses the selected
prompt file's containing directory as its root. Every content path must be
relative and is opened from its exact parsed text after a separate blank check;
contained parent components and whitespace-bearing names remain valid.
Exact prompt inspection performs one point-in-time lookup through that same
repository and validates referenced message content before returning declared
metadata. It does not parse templates or read profile, input, or schema
sources, and it does not retain the definition for a later execution.
Its package tests own prompt selection, strict decoding, definition validation,
duplicate detection, and source containment:
[prompt-definition repository tests](../../internal/promptdef/repository_test.go).
The runner selects from this index for preparation and `InspectPrompt`.
Neither path rescans configuration or reopens message files. `InspectPrompt`
returns declared metadata without rendering or resolving profile, input, or
schema resources. Runtime rendering still checks request variables and input
helpers. The [prompt adapter tests](../../internal/configuration/prompts_test.go)
cover resource origins, context containment, capture, and selection.
## Profiles And Built-Ins
`internal/profile` owns immutable in-memory and filesystem repositories, local
validation, overlays, and resolution for execution profiles. In-memory
definitions are normalized and validated once at that repository boundary and
are defensively copied on storage and lookup. A file contains
exactly one YAML document and its trimmed YAML `id` is its only selection
identity; filenames do not confer authority. Each point lookup reads discovered
files once for their metadata and reuses the selected file's bytes for strict
decoding; unrelated profiles are not fully decoded. Strict selected decoding
recognizes `base_profile` and the optional `backend` field, trims their values,
and permits inherited target fields only when a base is named. File-backed
`extra_params` values are validated and defensively copied through the shared
bounded JSON-value owner before a profile is published. OpenAI-compatible
reserved-field policy remains with the model-client and backend-registry owners.
`LoadFSRepository` is the eager immutable loading boundary for internal
catalog consumers. It discovers and strictly validates every raw profile once,
preserves safe source metadata including explicitly present YAML fields, and
publishes independently copied values from memory. It does not resolve profile
inheritance. Configured consumer sources continue to use the lazy point lookup
repositories described above.
`internal/catalog` selects the released keyed OpenRouter and Rakestrawhome
assets as one ConfigKit underlay and PromptKit-owned profiles as the next.
It discovers YAML inputs and supplies passive filesystem context without
parsing or pre-validating catalog definitions. The configuration loader
validates only the final effective backend and profile values, including
consumer overlays of maintained bases or backends. The new path has no catalog
source-layout, ownership-prefix, connection-field, or secret-key rejection.
consumer overlays of maintained bases or backends. There is no catalog-only
preflight, reserved source namespace, or replay of backend registration.
Until public engine assembly switches to that loader, `internal/legacycatalog`
supports the existing public construction path with the published flat `/v2`
assets. The repository behavior below describes that temporary path.
The overlay repository consults the next repository only when the
higher-precedence repository reports that a profile is absent. A reliably
selected malformed profile stops fallback, while an unrelated malformed file
does not become authoritative through its filename. Loading does not check
backend registry membership because the available registry belongs to the
assembled engine; the runner checks membership during preparation and exact
profile inspection.
The internal engine assembles one raw composite catalog in precedence order:
in-memory profiles, one ordinary configured source, an application fallback
source, Promptkit-owned profiles, then the maintained external catalog. It
constructs the owned definitions through the in-memory repository and validates
their maintained-only resolution and backend membership before consumer sources
are overlaid. An explicit file or `fs.FS` profile source replaces
`Config.ProfileDir` within the ordinary configured-source category. One outer
resolving repository wraps that complete raw catalog, so each base lookup
observes the same precedence and shadowing rules.
The resolving repository traverses every selected chain afresh, retains no
cache, detects cycles, limits a chain to 32 profiles, merges root-to-leaf into a
new caller-owned value, and validates the final target before publishing it. It
does not check backend registry membership. Exact `base_profile` syntax, merge
rules, and consumer-visible failure behavior belong to the [framework format
reference](../formats.md#profile-inheritance).
Exact profile inspection performs one point-in-time resolved lookup through
those profile sources and checks the final target without reading prompt, input,
or schema sources. It does not retain that lookup for a later execution.
Prepared execution instead freezes the fully resolved target; a later ordinary
operation performs a fresh traversal.
The imported external catalog provides external profiles and matching backend
definitions. The internal engine owns the Promptkit profiles documented in the
[framework format reference](../formats.md#built-in-profile-catalog). Profile
loading and overlay behavior are owned by the [profile repository tests](../../internal/profile/repository_test.go), while
catalog completeness, backend selection, and duplicate IDs are owned by the
[catalog adapter tests](../../internal/catalog/catalog_test.go).
ConfigKit resolves profile inheritance before the handoff. The configuration
adapter validates effective references and field values, then builds target
snapshots with backend defaults and presence/suppression state. A profile can
remain partial until runtime overrides complete it. `InspectProfile` selects
that captured target without prompt, artifact, schema, credential lookup, or
provider work. Preparation applies runtime overrides and then checks target
completeness. The [target tests](../../internal/configuration/targets_test.go)
cover effective target interpretation, while the [catalog tests](../../internal/catalog/catalog_test.go)
cover selected released assets.
## Ordinary Artifacts
@@ -161,14 +88,15 @@ implemented reader behavior and failures.
## Rendering
`internal/prompt` renders definition messages as Go templates using named
artifacts and variables. Within one render, each referenced artifact body is
converted to text lazily and cached by input name for reuse across the session
and every message; the cache is not shared across renders. Conversion uses
bounded chunks and preserves the artifact bytes exactly.
`internal/prompt` renders captured, pre-parsed definition templates using named
artifacts and variables. It clones templates to bind request-specific input
helpers without mutating shared state. Within one render, each referenced
artifact body is converted to text lazily and cached by input name for reuse
across the session and every message; the cache is not shared across renders.
Conversion uses bounded chunks and preserves the artifact bytes exactly.
Session and message parsing and execution remain synchronous. The renderer
checks cancellation before and after each parse and execution boundary,
Template parsing occurs during engine construction. Runtime execution remains
synchronous. The renderer checks cancellation around execution boundaries,
between artifact conversion chunks, around each message, and before publishing
the complete prompt. It cannot interrupt template work already in progress and
never publishes a partial prompt after observing cancellation. It validates and
@@ -205,9 +133,10 @@ not detach dependency work into background goroutines.
`Prepare` discards its validation plan after returning metadata. `Run` retains
its plan for initial and repaired-output validation, then discards it with the
operation. `PrepareExecution` retains the plan in its private frozen payload;
`RunPrepared` uses that plan without reopening prompt, profile, input, or
schema sources or rerendering the request. A later ordinary `Run` always
performs fresh source resolution and preparation.
`RunPrepared` uses that plan without reopening configuration, message, input,
or schema resources or rerendering the request. A later ordinary `Run` uses
the same captured configuration but prepares request artifacts and schemas
afresh.
The [validator tests](../../internal/validate/standard_validator_test.go) own
basic, JSON, JSON Schema, source resolution, schema loading, compilation,

32
docs/oss/attribution.md
View File

@@ -30,21 +30,6 @@ License and notices: [LGPL 3.0](../../LICENSE) |
[Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd).
## Promptkit OpenRouter Catalog v2
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2`
at `v2.20260926.0`.
Copyright (C) 2026 Eric Rakestraw
SPDX-License-Identifier: MIT
License and notices: [MIT](licenses/MIT.txt) |
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/4f731d217e0bc39151c010f50fb5912706e7b6b3/COPYRIGHT) |
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/4f731d217e0bc39151c010f50fb5912706e7b6b3/LICENSE).
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/4f731d217e0bc39151c010f50fb5912706e7b6b3).
## Promptkit OpenRouter Catalog v3
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3`
@@ -60,21 +45,6 @@ License and notices: [MIT](licenses/MIT.txt) |
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0).
## Promptkit Rakestrawhome Catalog v2
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2`
at `v2.20260926.0`.
Copyright (C) 2026 Eric Rakestraw
SPDX-License-Identifier: MIT
License and notices: [MIT](licenses/MIT.txt) |
[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/48918a3ad39fd99843cda94a86133b1b3881d596/COPYRIGHT) |
[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/48918a3ad39fd99843cda94a86133b1b3881d596/LICENSE).
[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/48918a3ad39fd99843cda94a86133b1b3881d596).
## Promptkit Rakestrawhome Catalog v3
Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3`
@@ -117,7 +87,7 @@ License and notices: [BSD 3-Clause](licenses/BSD-3-Clause.txt) |
## Go YAML v3
Selected module: `gopkg.in/yaml.v3` at `v3.0.1`.
Selected module: `gopkg.in/yaml.v3` at `v3.0.1` (transitive dependency).
Copyright (c) 2006-2010 Kirill Simonov

40
docs/policy/architecture.md
View File

@@ -51,23 +51,22 @@ The [internal component overview](../internal/overview.md) owns the inventory
of implemented packages and their concrete responsibilities. This policy owns
the boundaries and dependency rules those components must preserve.
The internal engine assembles one immutable backend registry, one capacity manager,
the internal repositories, renderer, validator, outbound client, and use-case
runner. The registry contains validated maintained definitions plus engine-scoped
consumer additions. The root facade translates public values and errors at the
library boundary.
The internal engine constructs the capacity manager from the registry's immutable
policy snapshot, wraps the selected built-in or injected model client, and
supplies bounded admission to the runner. The defaults and renderer depend on
the domain model. Prompt-definition and profile repositories use the domain
model, file catalog, and YAML decoder. The catalog adapter validates imported
immutable maintained data, including its reserved namespace, before internal
engine assembly supplies it to the profile package. Artifact reading uses the
domain model and application-neutral defaults. Validation uses the domain model,
file catalog, and JSON Schema implementation. The model client uses the domain
model and application-neutral defaults, and an injected or standard-library HTTP
client. The use-case runner depends on the narrow interfaces owned by each
internal component, including backend lookup and run admission.
The internal engine composes selected catalog and caller ConfigKit inputs once,
then assembles the captured definitions, capacity manager, renderer, validator,
outbound client, and use-case runner. The configuration adapter interprets and
validates only the final effective values, including presence, null, and
resource context. It retains source history for explicit operator inspection,
not for validation or a second resolution pass. The root facade translates
public values and errors at the library boundary.
The engine constructs capacity policy from final backend definitions, wraps
the selected built-in or injected model client, and supplies bounded admission
to the runner. Runtime/domain components consume execution-owned values, not
ConfigKit types. Artifact reading and validation use the domain model and
application-neutral defaults; validation also uses file-catalog helpers and
JSON Schema. The model client uses the domain model, defaults, and an injected
or standard-library HTTP client. The runner coordinates narrow lookup and
collaborator interfaces without rereading configuration.
The current implementation follows this dependency direction:
@@ -84,9 +83,10 @@ downstream consumers, including Scriptorium
narrow injected abstractions
```
The backend registry depends on the domain model and shared JSON-value
validation, has no mutation API after construction, and consumes the
OpenAI-compatible reserved request-field rule owned by the model client. The
The backend index depends on the domain model and shared JSON-value ownership
and has no mutation API after construction. The configuration adapter applies
the OpenAI-compatible reserved request-field rule owned by the model client
when it validates final provider maps. The
capacity component depends on the domain model and the narrow internal
model-client boundary, not on provider transport implementation. The model
client does not depend on registry or capacity configuration. The facade

2
docs/roadmap/implementation.md
View File

@@ -570,6 +570,8 @@ published, and no speculative release-note version is invented.
## Stage 11 — Remove Superseded Machinery And Consolidate Tests
**Completed:** 2026-09-30.
Scope: dead loading/resolution code and its obsolete tests, module cleanup, and
internal architectural documentation.

3598
engine_test.go
View File

File diff suppressed because it is too large Load Diff

10
errors.go
View File

@@ -72,10 +72,6 @@ func publicErrorFor(err error) error {
return ErrPromptLoad
case errors.Is(err, usecase.ErrProfileLoad):
return ErrProfileLoad
case errors.Is(err, promptdef.ErrInvalidYAML), errors.Is(err, promptdef.ErrInvalidPromptDefinition):
return ErrPromptLoad
case isProfileLoadCause(err):
return ErrProfileLoad
case errors.Is(err, usecase.ErrAPIKeyEnvMissing):
return errors.Join(ErrInvalidRequest, ErrAPIKeyEnvMissing)
case errors.Is(err, capacity.ErrCapacityExceeded):
@@ -94,9 +90,3 @@ func publicErrorFor(err error) error {
return nil
}
}
func isProfileLoadCause(err error) bool {
return errors.Is(err, profile.ErrInvalidYAML) ||
errors.Is(err, profile.ErrInvalidProfile) ||
errors.Is(err, profile.ErrRawAPIKeyNotAllowed)
}

45
generation_error_contract_test.go
View File

@@ -10,7 +10,6 @@ import (
"testing"
"gitea.maximumdirect.net/eric/promptkit"
"gitea.maximumdirect.net/eric/promptkit/testfixture"
)
func TestBuiltInGenerationError(t *testing.T) {
@@ -22,14 +21,14 @@ func TestBuiltInGenerationError(t *testing.T) {
engine := newBuiltInGenerationErrorEngine(t, http.StatusUnprocessableEntity,
`{"error":{"code":"`+codeMarker+`","type":"`+typeMarker+`","message":"`+messageMarker+`"}}`)
result, err := engine.Run(context.Background(), generationErrorRunRequest())
result, err := engine.Run(context.Background(), simpleGenerationRequest())
if result != nil {
t.Fatalf("Run result = %#v, want nil", result)
}
assertGenerationError(t, err, http.StatusUnprocessableEntity, codeMarker, typeMarker, messageMarker)
preparedEngine := newBuiltInGenerationErrorEngine(t, http.StatusServiceUnavailable, `{"error":{}}`)
prepared, err := preparedEngine.PrepareExecution(context.Background(), generationErrorRunRequest())
prepared, err := preparedEngine.PrepareExecution(context.Background(), simpleGenerationRequest())
if err != nil {
t.Fatalf("PrepareExecution: %v", err)
}
@@ -44,7 +43,7 @@ func TestBuiltInGenerationErrorWithAppendedMessages(t *testing.T) {
const messageMarker = "combined-request-provider-marker"
engine := newBuiltInGenerationErrorEngine(t, http.StatusUnprocessableEntity,
`{"error":{"message":"`+messageMarker+`"}}`)
request := generationErrorRunRequest()
request := simpleGenerationRequest()
request.AppendedMessages = []promptkit.RenderedMessage{
{Role: promptkit.RoleAssistant, Content: "previous response"},
{Role: promptkit.RoleUser, Content: "consumer correction"},
@@ -64,7 +63,7 @@ func TestBuiltInRepairGenerationError(t *testing.T) {
messageMarker = "repair-message-marker"
)
calls := 0
config := contractConfig(frameworkSchemaDir)
config := simpleConfig("message")
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
calls++
if calls == 1 {
@@ -74,11 +73,11 @@ func TestBuiltInRepairGenerationError(t *testing.T) {
body := `{"error":{"code":"` + codeMarker + `","type":"` + typeMarker + `","message":"` + messageMarker + `"}}`
return &http.Response{StatusCode: http.StatusUnprocessableEntity, ContentLength: int64(len(body)), Body: io.NopCloser(strings.NewReader(body))}, nil
})}
engine, err := testfixture.NewEngine(config)
engine, err := promptkit.NewEngine(config)
if err != nil {
t.Fatalf("NewEngine: %v", err)
}
req := generationErrorRunRequest()
req := simpleGenerationRequest()
req.Validation = &promptkit.OutputContract{
Format: promptkit.FormatJSON,
ValidationMode: promptkit.ValidationJSON,
@@ -97,15 +96,15 @@ func TestBuiltInRepairGenerationError(t *testing.T) {
func TestBuiltInGenerationFailureCategories(t *testing.T) {
transportCause := errors.New("private transport diagnostic")
config := contractConfig(frameworkSchemaDir)
config := simpleConfig("message")
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
return nil, transportCause
})}
engine, err := testfixture.NewEngine(config)
engine, err := promptkit.NewEngine(config)
if err != nil {
t.Fatal(err)
}
result, err := engine.Run(context.Background(), generationErrorRunRequest())
result, err := engine.Run(context.Background(), simpleGenerationRequest())
if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, promptkit.ErrTransport) || !errors.Is(err, transportCause) {
t.Fatalf("transport result=%#v, error=%v", result, err)
}
@@ -123,7 +122,7 @@ func TestBuiltInGenerationFailureCategories(t *testing.T) {
} {
t.Run(tc.name, func(t *testing.T) {
engine := newBuiltInGenerationErrorEngine(t, http.StatusOK, tc.body)
result, err := engine.Run(context.Background(), generationErrorRunRequest())
result, err := engine.Run(context.Background(), simpleGenerationRequest())
if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, promptkit.ErrMalformedResponse) || errors.Is(err, promptkit.ErrTransport) {
t.Fatalf("malformed result=%#v, error=%v", result, err)
}
@@ -140,15 +139,15 @@ func TestBuiltInGenerationPreservesContextIdentity(t *testing.T) {
{name: "deadline", cause: context.DeadlineExceeded},
} {
t.Run(tc.name, func(t *testing.T) {
config := contractConfig(frameworkSchemaDir)
config := simpleConfig("message")
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
return nil, tc.cause
})}
engine, err := testfixture.NewEngine(config)
engine, err := promptkit.NewEngine(config)
if err != nil {
t.Fatal(err)
}
result, err := engine.Run(context.Background(), generationErrorRunRequest())
result, err := engine.Run(context.Background(), simpleGenerationRequest())
if result != nil || !errors.Is(err, tc.cause) || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, promptkit.ErrTransport) {
t.Fatalf("context result=%#v, error=%v", result, err)
}
@@ -166,8 +165,8 @@ func TestInjectedGenerationFailureCategories(t *testing.T) {
{name: "arbitrary error", client: &fakeLLMClient{err: errors.New("injected failure")}},
} {
t.Run(tc.name, func(t *testing.T) {
engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(tc.client))
result, err := engine.Run(context.Background(), generationErrorRunRequest())
engine := newSimpleEngine(t, "message", promptkit.WithLLMClient(tc.client))
result, err := engine.Run(context.Background(), simpleGenerationRequest())
if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) || errors.Is(err, promptkit.ErrTransport) || errors.Is(err, promptkit.ErrMalformedResponse) != tc.malformed {
t.Fatalf("injected result=%#v, error=%v", result, err)
}
@@ -208,7 +207,7 @@ func assertGenerationError(t *testing.T, err error, statusCode int, code, provid
func newBuiltInGenerationErrorEngine(t *testing.T, statusCode int, body string) *promptkit.Engine {
t.Helper()
config := contractConfig(frameworkSchemaDir)
config := simpleConfig("message")
noRetries := 0
config.TransientRetries = &noRetries
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
@@ -218,19 +217,13 @@ func newBuiltInGenerationErrorEngine(t *testing.T, statusCode int, body string)
Body: io.NopCloser(strings.NewReader(body)),
}, nil
})}
engine, err := testfixture.NewEngine(config)
engine, err := promptkit.NewEngine(config)
if err != nil {
t.Fatalf("NewEngine: %v", err)
}
return engine
}
func generationErrorRunRequest() promptkit.RunRequest {
return promptkit.RunRequest{
PromptID: frameworkMarkdownSummaryPromptID,
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline("Rin opens the gate."),
"glossary": promptkit.Inline("gate: A guarded passage."),
},
}
func simpleGenerationRequest() promptkit.RunRequest {
return promptkit.RunRequest{PromptID: "prompt"}
}

3
go.mod
View File

@@ -4,12 +4,9 @@ go 1.25.5
require (
gitea.maximumdirect.net/eric/ConfigKit v0.5.0
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260926.0
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260926.0
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0
github.com/santhosh-tekuri/jsonschema/v6 v6.0.2
gopkg.in/yaml.v3 v3.0.1
)
require (

6
go.sum
View File

@@ -1,11 +1,7 @@
gitea.maximumdirect.net/eric/ConfigKit v0.5.0 h1:2KwG10+kApWM3avOoYA+p768xcbHUp7tQ2rqizBNpyg=
gitea.maximumdirect.net/eric/ConfigKit v0.5.0/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk=
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260926.0 h1:wdab+R+lgMZxzYwhbAuCrW7zdSQ1cW9ovBR6UzLmZR4=
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260926.0/go.mod h1:a3VBbh9oaGXD6ruxhtwHj+AWW6DY/SSyby1QtMmYSUo=
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 h1:lzfCDbzJQRGzIlRPRprnAV9hVVL5zUQy80VMQ0kbYa0=
gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0/go.mod h1:HjS0f+3q0IdHCTEz+VVAEgIwIWJFAZUjuKdAtvtqKdg=
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260926.0 h1:baCCIB6Ee5t0S4Zc++NTBIPcJEv5UPS6DlSrpZfykwg=
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260926.0/go.mod h1:mtgD1E0SGm0e/I8Ffhz+AYX7GN7RKUWJXehNFIKHs4Y=
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 h1:h12ZTRldAmpLAomj0oQRGteFxHLN1LPE6XDl9zJNbm8=
gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0/go.mod h1:OEFBeKdjTsLzqJwLmXI2O5gG3r64FktcVJVb3mBrRfg=
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
@@ -16,7 +12,5 @@ go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4=
go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

121
internal/backend/registry.go
View File

@@ -5,31 +5,21 @@ package backend
import (
"errors"
"fmt"
"regexp"
"sort"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
)
const (
// OpenRouterID is the reserved ID of Promptkit's built-in OpenRouter
// backend.
// OpenRouterID identifies the maintained OpenRouter backend.
OpenRouterID = "openrouter"
// RakestrawHomeID is the reserved ID of Promptkit's built-in Rakestrawhome
// backend.
// RakestrawHomeID identifies the maintained Rakestrawhome backend.
RakestrawHomeID = "rakestrawhome"
defaultQueueCapacity = 1024
)
// ErrBackendNotFound identifies a registry lookup for an unknown backend ID.
var ErrBackendNotFound = errors.New("backend not found")
var environmentVariableName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
// Registry is an immutable collection of validated backend definitions.
type Registry struct {
backends map[string]domain.Backend
@@ -52,36 +42,6 @@ func NewEffectiveRegistry(definitions map[string]domain.Backend) (*Registry, err
return registry, nil
}
// NewRegistry constructs a registry containing maintained definitions followed
// by consumer additions. Every ID must be unique across both groups.
func NewRegistry(maintained, additions []domain.Backend) (*Registry, error) {
registry := &Registry{
backends: make(map[string]domain.Backend, len(maintained)+len(additions)),
}
definitions := make([]domain.Backend, 0, len(maintained)+len(additions))
definitions = append(definitions, maintained...)
definitions = append(definitions, additions...)
for _, definition := range definitions {
definition.ID = strings.TrimSpace(definition.ID)
if definition.ID == "" {
return nil, errors.New("backend ID must not be blank")
}
if _, exists := registry.backends[definition.ID]; exists {
return nil, fmt.Errorf("backend ID %q is already registered", definition.ID)
}
normalized, err := NormalizeDefinition(definition)
if err != nil {
return nil, err
}
registry.backends[normalized.ID] = normalized
}
return registry, nil
}
// GetBackend returns a defensive copy of the backend registered with id.
func (r *Registry) GetBackend(id string) (domain.Backend, error) {
if r == nil {
@@ -117,80 +77,3 @@ func (r *Registry) CapacityPolicies() map[string]domain.BackendCapacityPolicy {
}
return policies
}
// NormalizeDefinition validates and defensively copies one backend definition.
func NormalizeDefinition(definition domain.Backend) (domain.Backend, error) {
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(definition.Endpoint)
if err != nil {
return domain.Backend{}, fmt.Errorf("backend %q endpoint: %w", definition.ID, err)
}
definition.Endpoint = endpoint
definition.APIKeyEnv = strings.TrimSpace(definition.APIKeyEnv)
if definition.APIKeyEnv != "" && !environmentVariableName.MatchString(definition.APIKeyEnv) {
return domain.Backend{}, fmt.Errorf(
"backend %q api key environment variable %q is invalid",
definition.ID,
definition.APIKeyEnv,
)
}
if definition.ConcurrencyLimit < 0 {
return domain.Backend{}, fmt.Errorf(
"backend %q concurrency limit must not be negative",
definition.ID,
)
}
if definition.QueueCapacity < 0 {
return domain.Backend{}, fmt.Errorf(
"backend %q queue capacity must not be negative",
definition.ID,
)
}
if definition.ConcurrencyLimit == 0 {
if definition.QueueCapacitySet {
return domain.Backend{}, fmt.Errorf(
"backend %q queue capacity requires a positive concurrency limit",
definition.ID,
)
}
definition.QueueCapacity = 0
} else {
if !definition.QueueCapacitySet {
definition.QueueCapacity = defaultQueueCapacity
definition.QueueCapacitySet = true
}
maxInt := int(^uint(0) >> 1)
if definition.QueueCapacity > maxInt-definition.ConcurrencyLimit {
return domain.Backend{}, fmt.Errorf(
"backend %q total capacity overflows int",
definition.ID,
)
}
}
keys := make([]string, 0, len(definition.ExtraParams))
for key := range definition.ExtraParams {
keys = append(keys, key)
}
sort.Strings(keys)
for _, key := range keys {
if key == "" {
return domain.Backend{}, fmt.Errorf("backend %q extra parameter key must not be empty", definition.ID)
}
if llm.IsReservedOpenAIChatRequestField(key) {
return domain.Backend{}, fmt.Errorf(
"backend %q extra parameter %q collides with a reserved request field",
definition.ID,
key,
)
}
}
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
if err != nil {
return domain.Backend{}, fmt.Errorf("backend %q extra parameters: %w", definition.ID, err)
}
definition.ExtraParams = extraParams
return definition, nil
}

389
internal/backend/registry_test.go
View File

@@ -3,55 +3,19 @@ package backend_test
import (
"errors"
"reflect"
"strings"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
const validEndpoint = "https://backend.example/v1"
func TestRegistryIncludesMaintainedDefinitions(t *testing.T) {
maintained := []domain.Backend{
{ID: backend.OpenRouterID, Endpoint: validEndpoint, ConcurrencyLimit: 2, QueueCapacity: 3, QueueCapacitySet: true},
{ID: backend.RakestrawHomeID, Endpoint: "https://second.example/v1", ConcurrencyLimit: 4, QueueCapacity: 5, QueueCapacitySet: true},
}
registry, err := backend.NewRegistry(maintained, nil)
if err != nil {
t.Fatalf("construct registry: %v", err)
}
for _, expected := range maintained {
t.Run(expected.ID, func(t *testing.T) {
definition, err := registry.GetBackend(expected.ID)
if err != nil {
t.Fatalf("look up maintained definition: %v", err)
}
if !reflect.DeepEqual(definition, expected) {
t.Fatalf("unexpected maintained definition: %#v", definition)
}
})
}
policies := registry.CapacityPolicies()
if len(policies) != 2 ||
policies[backend.OpenRouterID] != (domain.BackendCapacityPolicy{
ConcurrencyLimit: 2,
QueueCapacity: 3,
}) ||
policies[backend.RakestrawHomeID] != (domain.BackendCapacityPolicy{
ConcurrencyLimit: 4,
QueueCapacity: 5,
}) {
t.Fatalf("unexpected built-in capacity policies: %#v", policies)
}
}
func TestEffectiveRegistryKeepsLiteralIDsAndOwnsValues(t *testing.T) {
func TestEffectiveRegistryOwnsDefinitionsAndCapacityPolicies(t *testing.T) {
params := map[string]any{"nested": map[string]any{"enabled": true}}
registry, err := backend.NewEffectiveRegistry(map[string]domain.Backend{
" local.v1 ": {Endpoint: validEndpoint, ExtraParams: params, ConcurrencyLimit: 2, QueueCapacitySet: true},
" local.v1 ": {
Endpoint: "https://backend.example/v1", ExtraParams: params,
ConcurrencyLimit: 2, QueueCapacity: 3, QueueCapacitySet: true,
},
})
if err != nil {
t.Fatal(err)
@@ -59,341 +23,34 @@ func TestEffectiveRegistryKeepsLiteralIDsAndOwnsValues(t *testing.T) {
params["nested"].(map[string]any)["enabled"] = false
selected, err := registry.GetBackend(" local.v1 ")
if err != nil || selected.ID != " local.v1 " || selected.ExtraParams["nested"].(map[string]any)["enabled"] != true {
t.Fatalf("effective registry changed identity or retained caller values: %+v, %v", selected, err)
}
if _, err := registry.GetBackend("local.v1"); !errors.Is(err, backend.ErrBackendNotFound) {
t.Fatalf("trimmed identity unexpectedly matched: %v", err)
t.Fatalf("effective backend was not captured under its literal ID: %+v, %v", selected, err)
}
selected.ExtraParams["nested"].(map[string]any)["enabled"] = false
again, _ := registry.GetBackend(" local.v1 ")
if again.ExtraParams["nested"].(map[string]any)["enabled"] != true {
t.Fatal("registry lookup exposed its stored values")
again, err := registry.GetBackend(" local.v1 ")
if err != nil || again.ExtraParams["nested"].(map[string]any)["enabled"] != true {
t.Fatalf("lookup exposed registry values: %+v, %v", again, err)
}
}
func TestRegistryNormalizesUniqueAdditionsAndIsolatesMutations(t *testing.T) {
nested := map[string]int{"limit": 2}
extraParams := map[string]any{
"count": int64(7),
"nested": nested,
}
registry, err := backend.NewRegistry(nil, []domain.Backend{
{
ID: " custom ",
Endpoint: " https://custom.example/openai/v1 ",
APIKeyEnv: " CUSTOM_API_KEY ",
ExtraParams: extraParams,
ConcurrencyLimit: 3,
QueueCapacity: 2,
QueueCapacitySet: true,
},
{
ID: "Custom",
Endpoint: validEndpoint,
},
})
if err != nil {
t.Fatalf("construct registry: %v", err)
}
nested["limit"] = 99
extraParams["added"] = true
got, err := registry.GetBackend("custom")
if err != nil {
t.Fatalf("look up custom backend: %v", err)
}
if got.ID != "custom" ||
got.Endpoint != "https://custom.example/openai/v1" ||
got.APIKeyEnv != "CUSTOM_API_KEY" ||
got.ConcurrencyLimit != 3 ||
got.QueueCapacity != 2 ||
!got.QueueCapacitySet {
t.Fatalf("unexpected normalized definition: %#v", got)
}
if count, ok := got.ExtraParams["count"].(int64); !ok || count != 7 {
t.Fatalf("integer type or value changed: %#v", got.ExtraParams["count"])
}
gotNested, ok := got.ExtraParams["nested"].(map[string]int)
if !ok || gotNested["limit"] != 2 {
t.Fatalf("container type or value changed: %#v", got.ExtraParams["nested"])
}
if _, exists := got.ExtraParams["added"]; exists {
t.Fatalf("registry retained caller map: %#v", got.ExtraParams)
}
gotNested["limit"] = 100
got.ExtraParams["added"] = true
again, err := registry.GetBackend("custom")
if err != nil {
t.Fatalf("look up custom backend again: %v", err)
}
if again.ExtraParams["nested"].(map[string]int)["limit"] != 2 {
t.Fatalf("lookup exposed registry nested map: %#v", again.ExtraParams)
}
if _, exists := again.ExtraParams["added"]; exists {
t.Fatalf("lookup exposed registry map: %#v", again.ExtraParams)
}
if _, err := registry.GetBackend("Custom"); err != nil {
t.Fatalf("backend IDs should be case-sensitive: %v", err)
}
policies := registry.CapacityPolicies()
if len(policies) != 1 {
t.Fatalf("unexpected capacity policy count: %#v", policies)
}
policies["custom"] = domain.BackendCapacityPolicy{}
delete(policies, "custom")
againPolicies := registry.CapacityPolicies()
if againPolicies["custom"] != (domain.BackendCapacityPolicy{
ConcurrencyLimit: 3,
QueueCapacity: 2,
if !reflect.DeepEqual(policies, map[string]domain.BackendCapacityPolicy{
" local.v1 ": {ConcurrencyLimit: 2, QueueCapacity: 3},
}) {
t.Fatalf("capacity policy map mutated registry state: %#v", againPolicies)
t.Fatalf("capacity policy = %+v", policies)
}
delete(policies, " local.v1 ")
if len(registry.CapacityPolicies()) != 1 {
t.Fatal("capacity policy map exposed registry state")
}
}
func TestNewRegistryNormalizesCapacityPolicy(t *testing.T) {
maxInt := int(^uint(0) >> 1)
tests := []struct {
name string
definition domain.Backend
want domain.BackendCapacityPolicy
wantSet bool
wantError bool
}{
{
name: "unlimited when omitted",
definition: domain.Backend{},
},
{
name: "default queue",
definition: domain.Backend{
ConcurrencyLimit: 2,
},
want: domain.BackendCapacityPolicy{
ConcurrencyLimit: 2,
QueueCapacity: 1024,
},
wantSet: true,
},
{
name: "explicit zero queue",
definition: domain.Backend{
ConcurrencyLimit: 2,
QueueCapacitySet: true,
},
want: domain.BackendCapacityPolicy{
ConcurrencyLimit: 2,
},
wantSet: true,
},
{
name: "negative concurrency limit",
definition: domain.Backend{
ConcurrencyLimit: -1,
},
wantError: true,
},
{
name: "negative queue capacity",
definition: domain.Backend{
ConcurrencyLimit: 1,
QueueCapacity: -1,
QueueCapacitySet: true,
},
wantError: true,
},
{
name: "queue without limit",
definition: domain.Backend{
QueueCapacitySet: true,
},
wantError: true,
},
{
name: "total overflow",
definition: domain.Backend{
ConcurrencyLimit: maxInt,
QueueCapacity: 1,
QueueCapacitySet: true,
},
wantError: true,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
tc.definition.ID = "custom"
tc.definition.Endpoint = validEndpoint
registry, err := backend.NewRegistry(nil, []domain.Backend{tc.definition})
if tc.wantError {
if err == nil {
t.Fatal("expected invalid capacity policy error")
}
return
}
if err != nil {
t.Fatalf("construct registry: %v", err)
}
definition, err := registry.GetBackend("custom")
if err != nil {
t.Fatalf("look up custom backend: %v", err)
}
if definition.ConcurrencyLimit != tc.want.ConcurrencyLimit ||
definition.QueueCapacity != tc.want.QueueCapacity ||
definition.QueueCapacitySet != tc.wantSet {
t.Fatalf("normalized capacity=(%d, %d, %t), want (%d, %d, %t)",
definition.ConcurrencyLimit,
definition.QueueCapacity,
definition.QueueCapacitySet,
tc.want.ConcurrencyLimit,
tc.want.QueueCapacity,
tc.wantSet,
)
}
policies := registry.CapacityPolicies()
got, ok := policies["custom"]
if ok != tc.wantSet || got != tc.want {
t.Fatalf("capacity policy=(%#v, %t), want (%#v, %t)", got, ok, tc.want, tc.wantSet)
}
})
}
}
func TestNewRegistryRejectsDuplicateIDs(t *testing.T) {
tests := []struct {
name string
additions []domain.Backend
wantID string
}{
{
name: "OpenRouter collision after normalization",
additions: []domain.Backend{{ID: " openrouter "}},
wantID: backend.OpenRouterID,
},
{
name: "Rakestrawhome collision after normalization",
additions: []domain.Backend{{ID: " rakestrawhome "}},
wantID: backend.RakestrawHomeID,
},
{
name: "consumer collision after normalization",
additions: []domain.Backend{
{ID: "custom", Endpoint: validEndpoint},
{ID: " custom ", Endpoint: "https://other.example/v1"},
},
wantID: "custom",
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, err := backend.NewRegistry(nil, tc.additions)
if err == nil {
t.Fatal("expected duplicate ID error")
}
if !strings.Contains(err.Error(), tc.wantID) {
t.Fatalf("expected error to identify %q, got %v", tc.wantID, err)
}
})
}
}
func TestNewRegistryValidatesIDs(t *testing.T) {
for _, id := range []string{"", " \t\n "} {
t.Run(id, func(t *testing.T) {
_, err := backend.NewRegistry(nil, []domain.Backend{{
ID: id,
Endpoint: validEndpoint,
}})
if err == nil {
t.Fatal("expected blank ID error")
}
})
}
}
func TestNewRegistryValidatesEndpoints(t *testing.T) {
tests := []struct {
name string
endpoint string
}{
{name: "blank", endpoint: ""},
{name: "relative", endpoint: "/v1"},
{name: "missing host", endpoint: "https:///v1"},
{name: "unsupported scheme", endpoint: "ftp://backend.example/v1"},
{name: "user information", endpoint: "https://user@backend.example/v1"},
{name: "query", endpoint: "https://backend.example/v1?mode=chat"},
{name: "empty query", endpoint: "https://backend.example/v1?"},
{name: "fragment", endpoint: "https://backend.example/v1#chat"},
{name: "empty fragment", endpoint: "https://backend.example/v1#"},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, err := backend.NewRegistry(nil, []domain.Backend{{
ID: "custom",
Endpoint: tc.endpoint,
}})
if err == nil {
t.Fatal("expected invalid endpoint error")
}
})
}
}
func TestNewRegistryValidatesEnvironmentVariableNames(t *testing.T) {
for _, name := range []string{"1API_KEY", "API-KEY", "API KEY", "ÅPI_KEY"} {
t.Run(name, func(t *testing.T) {
_, err := backend.NewRegistry(nil, []domain.Backend{{
ID: "custom",
Endpoint: validEndpoint,
APIKeyEnv: name,
}})
if err == nil {
t.Fatal("expected invalid environment-variable name error")
}
})
}
}
func TestNewRegistryRejectsInvalidAndReservedExtraParameters(t *testing.T) {
tests := []struct {
name string
extraParams map[string]any
}{
{name: "unsupported value", extraParams: map[string]any{"value": make(chan int)}},
{name: "reserved key", extraParams: map[string]any{"model": "override"}},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, err := backend.NewRegistry(nil, []domain.Backend{{
ID: "custom",
Endpoint: validEndpoint,
ExtraParams: tc.extraParams,
}})
if err == nil {
t.Fatal("expected invalid extra parameters error")
}
})
}
}
func TestRegistryLookupReportsNotFound(t *testing.T) {
registry, err := backend.NewRegistry(nil, nil)
func TestEffectiveRegistryMissingBackend(t *testing.T) {
registry, err := backend.NewEffectiveRegistry(nil)
if err != nil {
t.Fatalf("construct registry: %v", err)
t.Fatal(err)
}
_, err = registry.GetBackend("missing")
if !errors.Is(err, backend.ErrBackendNotFound) {
t.Fatalf("expected ErrBackendNotFound, got %v", err)
if _, err := registry.GetBackend("missing"); !errors.Is(err, backend.ErrBackendNotFound) {
t.Fatalf("missing backend error = %v", err)
}
if !strings.Contains(err.Error(), "missing") {
t.Fatalf("expected error to identify backend, got %v", err)
if _, err := registry.GetBackend("local.v1"); !errors.Is(err, backend.ErrBackendNotFound) {
t.Fatalf("trimmed lookup unexpectedly matched: %v", err)
}
}

3
internal/configuration/load.go
View File

@@ -109,8 +109,7 @@ func Load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error)
return load(main, underlays)
}
// load performs one ConfigKit lifecycle over the selected source tiers. Engine
// assembly is intentionally not connected to this adapter yet.
// load performs one ConfigKit lifecycle over the selected source tiers.
func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) {
state := &loaded{
backends: make(map[string]map[string]fieldState),

2
internal/configuration/targets.go
View File

@@ -103,7 +103,7 @@ func (state *loaded) buildExecutionProfiles() error {
}
target.ExtraParams = merged
}
profile := domain.ExecutionProfile{ID: id, BackendID: target.BackendID, ResolvedTarget: &target, TargetPresence: presence}
profile := domain.ExecutionProfile{ID: id, ResolvedTarget: &target, TargetPresence: presence}
state.indexedProfiles[id] = profile
}
return nil

23
internal/domain/domain.go
View File

@@ -192,26 +192,11 @@ type BackendCapacityPolicy struct {
QueueCapacity int
}
// ExecutionProfile describes how and where to execute a model.
// ExecutionProfile holds a captured configuration target for execution.
type ExecutionProfile struct {
ID string `yaml:"id"`
BaseProfileID string `yaml:"base_profile"`
BackendID string `yaml:"backend"`
Endpoint string `yaml:"endpoint"`
Model string `yaml:"model"`
Temperature float64 `yaml:"temperature"`
MaxTokens int `yaml:"max_tokens"`
TopP float64 `yaml:"top_p"`
TimeoutSeconds int `yaml:"timeout_seconds"`
ServiceTier string `yaml:"service_tier"`
ReasoningEffort string `yaml:"reasoning_effort"`
APIKeyEnv string `yaml:"api_key_env"`
APIKeyRequired bool `yaml:"-" json:"-"`
ExtraParams map[string]any `yaml:"extra_params"`
// ResolvedTarget is an execution-owned snapshot assembled from effective
// configuration and backend defaults. Legacy source repositories leave it nil.
ResolvedTarget *ExecutionTarget `yaml:"-" json:"-"`
TargetPresence ExecutionTargetPresence `yaml:"-" json:"-"`
ID string
ResolvedTarget *ExecutionTarget
TargetPresence ExecutionTargetPresence
}
// ExecutionTargetOverride represents per-request runtime setting overrides.

2
internal/engine/engine.go
View File

@@ -133,7 +133,7 @@ func New(cfg Config, options Options) (*Engine, error) {
}
return &Engine{
runner: usecase.NewRunnerWithRepairer(snapshot, snapshot, snapshot, reader, prompt.NewCompiledRenderer(snapshot), client, validator, usecase.NewDefaultOutputRepairer(client), manager, defaultRetries),
runner: usecase.NewRunnerWithRepairer(snapshot, snapshot, reader, prompt.NewCompiledRenderer(snapshot), client, validator, usecase.NewDefaultOutputRepairer(client), manager, defaultRetries),
inspection: snapshot.Inspection(),
}, nil
}

50
internal/engine/maintained_profiles.go
View File

@@ -1,50 +0,0 @@
package engine
import (
"context"
"fmt"
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
)
func maintainedProfileDefinitions() []domain.ExecutionProfile {
return []domain.ExecutionProfile{
{
ID: "promptkit-openrouter-standard",
BaseProfileID: "gemini-flash-latest",
},
{
ID: "promptkit-openrouter-auto",
BackendID: backend.OpenRouterID,
Model: "openrouter/auto",
},
}
}
func newMaintainedProfileRepository(catalogProfiles profile.Repository, maintainedBackends []domain.Backend) (profile.Repository, error) {
definitions := maintainedProfileDefinitions()
owned, err := profile.NewMemoryRepository(definitions)
if err != nil {
return nil, fmt.Errorf("create Promptkit-maintained profiles: %w", err)
}
combined := profile.NewOverlayRepository(owned, catalogProfiles)
resolved := profile.NewResolvingRepository(combined)
registry, err := backend.NewRegistry(maintainedBackends, nil)
if err != nil {
return nil, fmt.Errorf("construct maintained backend registry: %w", err)
}
for _, definition := range definitions {
effective, err := resolved.GetProfile(context.Background(), definition.ID)
if err != nil {
return nil, fmt.Errorf("resolve Promptkit-maintained profile %q: %w", definition.ID, err)
}
if _, err := registry.GetBackend(effective.BackendID); err != nil {
return nil, fmt.Errorf("Promptkit-maintained profile %q selects unavailable backend %q: %w", definition.ID, effective.BackendID, err)
}
}
return combined, nil
}

78
internal/engine/maintained_profiles_test.go
View File

@@ -1,78 +0,0 @@
package engine
import (
"context"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
)
func TestNewMaintainedProfileRepository(t *testing.T) {
newCatalogProfiles := func(t *testing.T, definitions []domain.ExecutionProfile) profile.Repository {
t.Helper()
repository, err := profile.NewMemoryRepository(definitions)
if err != nil {
t.Fatalf("create catalog profiles: %v", err)
}
return repository
}
newBackend := func(id string) domain.Backend {
return domain.Backend{ID: id, Endpoint: "https://" + id + ".example/v1", APIKeyEnv: "TEST_API_KEY"}
}
t.Run("resolves inherited and direct definitions", func(t *testing.T) {
repository, err := newMaintainedProfileRepository(
newCatalogProfiles(t, []domain.ExecutionProfile{{
ID: "gemini-flash-latest",
BackendID: backend.OpenRouterID,
Model: "catalog-selected-model",
}}),
[]domain.Backend{newBackend(backend.OpenRouterID)},
)
if err != nil {
t.Fatalf("create maintained profiles: %v", err)
}
resolved := profile.NewResolvingRepository(repository)
standard, err := resolved.GetProfile(context.Background(), "promptkit-openrouter-standard")
if err != nil {
t.Fatalf("resolve inherited profile: %v", err)
}
if standard.BackendID != backend.OpenRouterID || standard.Model != "catalog-selected-model" {
t.Fatalf("unexpected inherited profile: %#v", standard)
}
auto, err := resolved.GetProfile(context.Background(), "promptkit-openrouter-auto")
if err != nil {
t.Fatalf("resolve direct profile: %v", err)
}
definitions := maintainedProfileDefinitions()
if auto.BackendID != definitions[1].BackendID || auto.Model != definitions[1].Model {
t.Fatalf("unexpected direct profile: %#v", auto)
}
})
t.Run("rejects a missing inherited base", func(t *testing.T) {
if _, err := newMaintainedProfileRepository(
newCatalogProfiles(t, nil),
[]domain.Backend{newBackend(backend.OpenRouterID)},
); err == nil {
t.Fatal("expected missing base error")
}
})
t.Run("rejects an unavailable effective backend", func(t *testing.T) {
if _, err := newMaintainedProfileRepository(
newCatalogProfiles(t, []domain.ExecutionProfile{{
ID: "gemini-flash-latest",
BackendID: "other",
Model: "catalog-selected-model",
}}),
[]domain.Backend{newBackend("other")},
); err == nil {
t.Fatal("expected unavailable backend error")
}
})
}

250
internal/legacycatalog/catalog.go
View File

@@ -1,250 +0,0 @@
// Package legacycatalog supports the existing public construction path while
// catalog configuration moves to the effective ConfigKit loader.
package legacycatalog
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"path"
"sort"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
)
// Source identifies one immutable backend catalog asset tree.
type Source struct {
Name string
ExpectedBackendID string
FS fs.FS
Root string
}
// Set is the validated maintained backend and raw profile catalog.
type Set struct {
Backends []domain.Backend
Profiles profile.Repository
profileIDs []string
}
// Load validates and combines immutable catalog sources in source order.
func Load(sources ...Source) (Set, error) {
if len(sources) == 0 {
return Set{}, errors.New("at least one catalog source is required")
}
loaded := Set{Backends: make([]domain.Backend, 0, len(sources))}
names := map[string]bool{}
backendIDs := map[string]bool{}
profileIDs := map[string]bool{}
for _, source := range sources {
if err := validateSource(source, names); err != nil {
return Set{}, err
}
names[source.Name] = true
definition, err := loadBackend(source)
if err != nil {
return Set{}, err
}
if backendIDs[definition.ID] {
return Set{}, fmt.Errorf("catalog %s: backend ID duplicates an earlier catalog", source.Name)
}
backendIDs[definition.ID] = true
repository, metadata, err := profile.LoadFSRepository(context.Background(), source.FS, path.Join(source.Root, "profiles"))
if err != nil {
return Set{}, fmt.Errorf("catalog %s: %w", source.Name, err)
}
if len(metadata) == 0 {
return Set{}, fmt.Errorf("catalog %s: profiles must not be empty", source.Name)
}
resolvingRepository := profile.NewResolvingRepository(repository)
for _, entry := range metadata {
if strings.HasPrefix(entry.ID, "promptkit-") {
return Set{}, fmt.Errorf("catalog %s: %s uses the reserved promptkit- profile ID prefix", source.Name, entry.Path)
}
if profileIDs[entry.ID] {
return Set{}, fmt.Errorf("catalog %s: %s duplicates an earlier profile ID", source.Name, entry.Path)
}
if containsField(entry.ExplicitFields, "endpoint") || containsField(entry.ExplicitFields, "api_key_env") {
return Set{}, fmt.Errorf("catalog %s: %s contains connection metadata", source.Name, entry.Path)
}
value, err := repository.GetProfile(context.Background(), entry.ID)
if err != nil {
return Set{}, fmt.Errorf("catalog %s: %s: %w", source.Name, entry.Path, err)
}
if err := rejectSecretKeys(value.ExtraParams); err != nil {
return Set{}, fmt.Errorf("catalog %s: %s: prohibited extra parameter key", source.Name, entry.Path)
}
resolved, err := resolvingRepository.GetProfile(context.Background(), entry.ID)
if err != nil {
return Set{}, fmt.Errorf("catalog %s: %s has invalid profile inheritance", source.Name, entry.Path)
}
if resolved.BackendID != definition.ID {
return Set{}, fmt.Errorf("catalog %s: %s selects a different backend", source.Name, entry.Path)
}
profileIDs[entry.ID] = true
loaded.profileIDs = append(loaded.profileIDs, entry.ID)
}
if loaded.Profiles == nil {
loaded.Profiles = repository
} else {
loaded.Profiles = profile.NewOverlayRepository(loaded.Profiles, repository)
}
loaded.Backends = append(loaded.Backends, definition)
}
sort.Strings(loaded.profileIDs)
return loaded, nil
}
func validateSource(source Source, names map[string]bool) error {
if strings.TrimSpace(source.Name) == "" || source.Name != strings.TrimSpace(source.Name) {
return errors.New("catalog source name must not be blank")
}
if names[source.Name] {
return fmt.Errorf("duplicate catalog source name %q", source.Name)
}
if source.FS == nil {
return fmt.Errorf("catalog %s: filesystem is nil", source.Name)
}
if source.Root == "." || source.Root == "" || !fs.ValidPath(source.Root) {
return fmt.Errorf("catalog %s: asset root is invalid", source.Name)
}
if strings.TrimSpace(source.ExpectedBackendID) == "" {
return fmt.Errorf("catalog %s: expected backend ID is blank", source.Name)
}
return validateLayout(source)
}
func validateLayout(source Source) error {
manifestPath := path.Join(source.Root, "backend.json")
profilesRoot := path.Join(source.Root, "profiles")
return fs.WalkDir(source.FS, source.Root, func(assetPath string, entry fs.DirEntry, err error) error {
if err != nil {
return err
}
if assetPath == source.Root {
if !entry.IsDir() {
return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath)
}
return nil
}
if entry.IsDir() {
if assetPath == profilesRoot || strings.HasPrefix(assetPath, profilesRoot+"/") {
return nil
}
return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath)
}
if assetPath == manifestPath && entry.Type().IsRegular() {
return nil
}
if strings.HasPrefix(assetPath, profilesRoot+"/") && entry.Type().IsRegular() && strings.HasSuffix(assetPath, ".yml") {
return nil
}
return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath)
})
}
func loadBackend(source Source) (domain.Backend, error) {
data, err := fs.ReadFile(source.FS, path.Join(source.Root, "backend.json"))
if err != nil {
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: %w", source.Name, err)
}
type manifest struct {
SchemaVersion *int `json:"schema_version"`
ID *string `json:"id"`
Endpoint *string `json:"endpoint"`
APIKeyEnv *string `json:"api_key_env"`
ConcurrencyLimit *int `json:"concurrency_limit"`
QueueCapacity *int `json:"queue_capacity"`
ExtraParams json.RawMessage `json:"extra_params"`
}
decoder := json.NewDecoder(bytes.NewReader(data))
decoder.DisallowUnknownFields()
var value manifest
if err := decoder.Decode(&value); err != nil {
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid manifest", source.Name)
}
var trailing any
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid manifest", source.Name)
}
if value.SchemaVersion == nil || *value.SchemaVersion != 1 || value.ID == nil || value.Endpoint == nil || value.APIKeyEnv == nil || value.ConcurrencyLimit == nil || value.QueueCapacity == nil || value.ExtraParams == nil {
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: required field is missing or unsupported", source.Name)
}
if *value.ID != source.ExpectedBackendID {
return domain.Backend{}, fmt.Errorf("catalog %s: backend ID does not match expected ID", source.Name)
}
if strings.TrimSpace(*value.APIKeyEnv) == "" {
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: api key environment variable must not be blank", source.Name)
}
extraParams, err := decodeExtraParams(value.ExtraParams)
if err != nil {
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid extra parameters", source.Name)
}
if err := rejectSecretKeys(extraParams); err != nil {
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: prohibited extra parameter key", source.Name)
}
normalized, err := backend.NormalizeDefinition(domain.Backend{ID: *value.ID, Endpoint: *value.Endpoint, APIKeyEnv: *value.APIKeyEnv, ExtraParams: extraParams, ConcurrencyLimit: *value.ConcurrencyLimit, QueueCapacity: *value.QueueCapacity, QueueCapacitySet: true})
if err != nil {
return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid backend definition", source.Name)
}
return normalized, nil
}
func decodeExtraParams(data []byte) (map[string]any, error) {
if bytes.Equal(bytes.TrimSpace(data), []byte("null")) {
return nil, nil
}
decoder := json.NewDecoder(bytes.NewReader(data))
decoder.UseNumber()
var value map[string]any
if err := decoder.Decode(&value); err != nil || value == nil {
return nil, errors.New("extra parameters must be an object or null")
}
var trailing any
if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) {
return nil, errors.New("extra parameters must contain exactly one JSON value")
}
return value, nil
}
func containsField(fields []string, target string) bool {
for _, field := range fields {
if field == target {
return true
}
}
return false
}
func rejectSecretKeys(value map[string]any) error {
return rejectSecretValue(value)
}
func rejectSecretValue(value any) error {
switch value := value.(type) {
case map[string]any:
for key, child := range value {
switch strings.ToLower(key) {
case "api_key", "apikey", "authorization", "credential", "credentials", "password", "secret", "token", "access_token":
return errors.New("prohibited key")
}
if err := rejectSecretValue(child); err != nil {
return err
}
}
case []any:
for _, child := range value {
if err := rejectSecretValue(child); err != nil {
return err
}
}
}
return nil
}

47
internal/profile/definition.go
View File

@@ -1,47 +0,0 @@
package profile
import (
"errors"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
// NormalizeAndValidateDefinition normalizes and validates one source-local
// profile definition without resolving a base profile.
func NormalizeAndValidateDefinition(profile *domain.ExecutionProfile) error {
if profile == nil {
return errors.New("profile is required")
}
profile.ID = strings.TrimSpace(profile.ID)
profile.BaseProfileID = strings.TrimSpace(profile.BaseProfileID)
profile.BackendID = strings.TrimSpace(profile.BackendID)
profile.Endpoint = strings.TrimSpace(profile.Endpoint)
if profile.ID == "" {
return errors.New("id is required")
}
if profile.Endpoint != "" {
endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(profile.Endpoint)
if err != nil {
return err
}
profile.Endpoint = endpoint
}
if profile.BaseProfileID == "" {
if profile.BackendID == "" && profile.Endpoint == "" {
return errors.New("backend or endpoint is required")
}
if strings.TrimSpace(profile.Model) == "" {
return errors.New("model is required")
}
}
return domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{
Temperature: profile.Temperature,
MaxTokens: profile.MaxTokens,
TopP: profile.TopP,
TimeoutSeconds: profile.TimeoutSeconds,
})
}

98
internal/profile/eager_repository.go
View File

@@ -1,98 +0,0 @@
package profile
import (
"context"
"fmt"
"io/fs"
"sort"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
)
// LoadedProfileMetadata identifies one profile accepted by LoadFSRepository.
type LoadedProfileMetadata struct {
// ID is the normalized profile ID.
ID string
// Path is the safe root-relative source path.
Path string
// ExplicitFields lists the sorted top-level YAML fields present in source.
ExplicitFields []string
}
// LoadFSRepository eagerly validates every profile under root and returns an
// immutable raw repository and independently owned source metadata.
func LoadFSRepository(ctx context.Context, fsys fs.FS, root string) (Repository, []LoadedProfileMetadata, error) {
if fsys == nil {
return nil, nil, fmt.Errorf("failed to read profile directory: filesystem is nil")
}
paths, err := filecatalog.FindFSYAMLFiles(ctx, fsys, root)
if err != nil {
return nil, nil, fmt.Errorf("failed to read profile directory: %w", err)
}
repository := &loadedRepository{profiles: make(map[string]domain.ExecutionProfile, len(paths))}
metadata := make([]LoadedProfileMetadata, 0, len(paths))
for _, path := range paths {
if err := ctx.Err(); err != nil {
return nil, nil, err
}
data, err := fs.ReadFile(fsys, path)
if err != nil {
return nil, nil, fmt.Errorf("failed to read profile file %s: %w", filecatalog.DisplayPath(root, path), err)
}
fileMetadata, err := readProfileFileMetadata(data)
if err != nil {
return nil, nil, fmt.Errorf("%w: %s", ErrInvalidYAML, filecatalog.DisplayPath(root, path))
}
if fileMetadata.hasRawAPIKey {
return nil, nil, fmt.Errorf("%w: %s", ErrRawAPIKeyNotAllowed, filecatalog.DisplayPath(root, path))
}
definition, err := decodeProfile(data)
if err != nil {
return nil, nil, fmt.Errorf("%w: %s", ErrInvalidYAML, filecatalog.DisplayPath(root, path))
}
definition.ExtraParams, err = jsonvalue.CopyMap(definition.ExtraParams)
if err != nil {
return nil, nil, fmt.Errorf("%w: %s", ErrInvalidProfile, filecatalog.DisplayPath(root, path))
}
if err := NormalizeAndValidateDefinition(definition); err != nil {
return nil, nil, fmt.Errorf("%w: %s", ErrInvalidProfile, filecatalog.DisplayPath(root, path))
}
if _, exists := repository.profiles[definition.ID]; exists {
return nil, nil, fmt.Errorf("%w: %s: duplicate profile ID", ErrInvalidProfile, filecatalog.DisplayPath(root, path))
}
repository.profiles[definition.ID] = *definition
fields := append([]string(nil), fileMetadata.explicitFields...)
sort.Strings(fields)
metadata = append(metadata, LoadedProfileMetadata{
ID: definition.ID,
Path: filecatalog.DisplayPath(root, path),
ExplicitFields: fields,
})
}
sort.Slice(metadata, func(left, right int) bool { return metadata[left].ID < metadata[right].ID })
return repository, metadata, nil
}
type loadedRepository struct {
profiles map[string]domain.ExecutionProfile
}
func (r *loadedRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
definition, found := r.profiles[strings.TrimSpace(id)]
if !found {
return nil, ErrProfileNotFound
}
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
if err != nil {
return nil, fmt.Errorf("copy loaded profile %q: %w", definition.ID, err)
}
definition.ExtraParams = extraParams
return &definition, nil
}

110
internal/profile/eager_repository_test.go
View File

@@ -1,110 +0,0 @@
package profile
import (
"context"
"errors"
"io/fs"
"reflect"
"testing"
"testing/fstest"
)
func TestLoadFSRepositoryLoadsSortedIndependentProfiles(t *testing.T) {
fsys := fstest.MapFS{
"profiles/z.yml": {Data: []byte("id: z\nbackend: local\nmodel: z-model\nextra_params:\n nested:\n value: one\n")},
"profiles/a.yml": {Data: []byte("id: a\nbackend: local\nmodel: a-model\nendpoint: ''\n")},
}
repository, metadata, err := LoadFSRepository(context.Background(), fsys, "profiles")
if err != nil {
t.Fatalf("load repository: %v", err)
}
if len(metadata) != 2 || metadata[0].ID != "a" || metadata[1].ID != "z" || metadata[0].Path != "a.yml" {
t.Fatalf("unexpected metadata: %#v", metadata)
}
if len(metadata[0].ExplicitFields) != 4 || metadata[0].ExplicitFields[0] != "backend" || metadata[0].ExplicitFields[1] != "endpoint" {
t.Fatalf("expected explicitly empty endpoint metadata, got %#v", metadata[0].ExplicitFields)
}
metadata[1].ExplicitFields[0] = "changed"
first, err := repository.GetProfile(context.Background(), "z")
if err != nil {
t.Fatalf("load profile: %v", err)
}
first.ExtraParams["nested"].(map[string]any)["value"] = "changed"
second, err := repository.GetProfile(context.Background(), "z")
if err != nil {
t.Fatalf("reload profile: %v", err)
}
if second.ExtraParams["nested"].(map[string]any)["value"] != "one" {
t.Fatalf("profile value was not defensively copied: %#v", second)
}
}
func TestLoadFSRepositoryRejectsInvalidProfiles(t *testing.T) {
tests := map[string]fstest.MapFS{
"duplicate IDs": {
"profiles/one.yml": {Data: []byte("id: duplicate\nbackend: local\nmodel: one\n")},
"profiles/two.yml": {Data: []byte("id: duplicate\nbackend: local\nmodel: two\n")},
},
"raw API key": {
"profiles/one.yml": {Data: []byte("id: one\nbackend: local\nmodel: one\napi_key: forbidden\n")},
},
"multiple documents": {
"profiles/one.yml": {Data: []byte("id: one\nbackend: local\nmodel: one\n---\nid: two\n")},
},
}
for name, fsys := range tests {
t.Run(name, func(t *testing.T) {
_, _, err := LoadFSRepository(context.Background(), fsys, "profiles")
if err == nil {
t.Fatal("expected load failure")
}
})
}
}
func TestLoadedRepositoryHonorsCancellationAndMissingProfiles(t *testing.T) {
repository, _, err := LoadFSRepository(context.Background(), fstest.MapFS{
"profiles/one.yml": {Data: []byte("id: one\nbackend: local\nmodel: one\n")},
}, "profiles")
if err != nil {
t.Fatalf("load repository: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := repository.GetProfile(ctx, "one"); !errors.Is(err, context.Canceled) {
t.Fatalf("expected cancellation, got %v", err)
}
if _, err := repository.GetProfile(context.Background(), "missing"); !errors.Is(err, ErrProfileNotFound) {
t.Fatalf("expected missing profile, got %v", err)
}
}
func TestLoadFSRepositoryMatchesPointLookupForRawProfiles(t *testing.T) {
fsys := fstest.MapFS{
"profiles/base.yml": {Data: []byte("id: base\nbackend: local\nmodel: base-model\n")},
"profiles/derived.yml": {Data: []byte("id: derived\nbase_profile: base\nreasoning_effort: high\n")},
}
eager, _, err := LoadFSRepository(context.Background(), fsys, "profiles")
if err != nil {
t.Fatalf("load eager repository: %v", err)
}
pointLookup := NewFSRepository(fsys, "profiles")
for _, id := range []string{"base", "derived"} {
t.Run(id, func(t *testing.T) {
got, err := eager.GetProfile(context.Background(), id)
if err != nil {
t.Fatalf("load eager profile: %v", err)
}
want, err := pointLookup.GetProfile(context.Background(), id)
if err != nil {
t.Fatalf("load point-in-time profile: %v", err)
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("raw profiles differ: got %#v, want %#v", got, want)
}
})
}
}
var _ fs.FS = fstest.MapFS{}

259
internal/profile/filesystem_repository.go
View File

@@ -1,259 +0,0 @@
package profile
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"io/fs"
"os"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
"gopkg.in/yaml.v3"
)
var (
ErrProfileNotFound = errors.New("execution profile not found")
ErrInvalidYAML = errors.New("invalid YAML format")
ErrInvalidProfile = errors.New("invalid execution profile configuration")
ErrRawAPIKeyNotAllowed = errors.New("raw api_key is not allowed; use api_key_env")
)
type filesystemRepository struct {
dir string
}
func NewFilesystemRepository(dir string) Repository {
return &filesystemRepository{dir: dir}
}
func (r *filesystemRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
return loadProfile(ctx, os.DirFS(r.dir), ".", id)
}
type fsRepository struct {
fsys fs.FS
root string
}
func NewFSRepository(fsys fs.FS, root string) Repository {
return &fsRepository{fsys: fsys, root: root}
}
func (r *fsRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
return loadProfile(ctx, r.fsys, r.root, id)
}
type overlayRepository struct {
primary Repository
fallback Repository
}
func NewOverlayRepository(primary, fallback Repository) Repository {
return &overlayRepository{primary: primary, fallback: fallback}
}
func (r *overlayRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
if r.primary != nil {
prof, err := r.primary.GetProfile(ctx, id)
if err == nil {
return prof, nil
}
if !errors.Is(err, ErrProfileNotFound) {
return nil, err
}
}
if r.fallback == nil {
return nil, ErrProfileNotFound
}
return r.fallback.GetProfile(ctx, id)
}
func loadProfile(ctx context.Context, fsys fs.FS, root string, id string) (*domain.ExecutionProfile, error) {
id = strings.TrimSpace(id)
if id == "" {
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile)
}
if fsys == nil {
return nil, fmt.Errorf("failed to read profile directory: filesystem is nil")
}
files, err := filecatalog.FindFSYAMLFiles(ctx, fsys, root)
if err != nil {
return nil, fmt.Errorf("failed to read profile directory: %w", err)
}
var matches []profileMatch
for _, fullPath := range files {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
relPath := filecatalog.DisplayPath(root, fullPath)
data, err := fs.ReadFile(fsys, fullPath)
if err != nil {
return nil, fmt.Errorf("failed to read profile file %s: %w", relPath, err)
}
metadata, metadataErr := readProfileFileMetadata(data)
idMatch := metadata.matchesID(id)
if metadataErr != nil {
if idMatch {
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, metadataErr)
}
continue
}
if metadata.hasRawAPIKey {
if idMatch {
return nil, fmt.Errorf("%w: %s", ErrRawAPIKeyNotAllowed, relPath)
}
continue
}
if !idMatch {
continue
}
prof, err := decodeProfile(data)
if err != nil {
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err)
}
prof.ID = strings.TrimSpace(prof.ID)
if prof.ID != id {
continue
}
prof.ExtraParams, err = jsonvalue.CopyMap(prof.ExtraParams)
if err != nil {
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidProfile, relPath, err)
}
if err := NormalizeAndValidateDefinition(prof); err != nil {
if errors.Is(err, ErrRawAPIKeyNotAllowed) {
return nil, fmt.Errorf("%w: %s", err, relPath)
}
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidProfile, relPath, err)
}
matches = append(matches, profileMatch{
profile: prof,
path: relPath,
})
}
if len(matches) > 1 {
paths := make([]string, 0, len(matches))
for _, match := range matches {
paths = append(paths, match.path)
}
return nil, fmt.Errorf("%w: duplicate execution profile id %q found in: %s", ErrInvalidProfile, id, strings.Join(paths, ", "))
}
if len(matches) == 1 {
return matches[0].profile, nil
}
return nil, ErrProfileNotFound
}
type profileMatch struct {
profile *domain.ExecutionProfile
path string
}
type profileFileMetadata struct {
ids []string
hasRawAPIKey bool
explicitFields []string
}
func readProfileFileMetadata(data []byte) (profileFileMetadata, error) {
decoder := yaml.NewDecoder(bytes.NewReader(data))
var node yaml.Node
if err := decoder.Decode(&node); err != nil {
return profileFileMetadata{}, err
}
metadata := profileMetadataFromNode(&node)
documentCount := 1
for {
var trailing yaml.Node
err := decoder.Decode(&trailing)
if errors.Is(err, io.EOF) {
if documentCount == 1 {
return metadata, nil
}
return metadata, errors.New("profile file must contain exactly one YAML document")
}
if err != nil {
return metadata, err
}
documentCount++
metadata.merge(profileMetadataFromNode(&trailing))
}
}
func profileMetadataFromNode(node *yaml.Node) profileFileMetadata {
if node.Kind != yaml.DocumentNode || len(node.Content) == 0 {
return profileFileMetadata{}
}
mapping := node.Content[0]
if mapping.Kind != yaml.MappingNode {
return profileFileMetadata{}
}
var metadata profileFileMetadata
for i := 0; i+1 < len(mapping.Content); i += 2 {
key := mapping.Content[i]
value := mapping.Content[i+1]
metadata.explicitFields = append(metadata.explicitFields, key.Value)
switch key.Value {
case "id":
metadata.ids = append(metadata.ids, strings.TrimSpace(value.Value))
case "api_key":
metadata.hasRawAPIKey = true
}
}
return metadata
}
func (m profileFileMetadata) matchesID(id string) bool {
for _, candidate := range m.ids {
if candidate == id {
return true
}
}
return false
}
func (m *profileFileMetadata) merge(other profileFileMetadata) {
m.ids = append(m.ids, other.ids...)
m.hasRawAPIKey = m.hasRawAPIKey || other.hasRawAPIKey
m.explicitFields = append(m.explicitFields, other.explicitFields...)
}
func decodeProfile(data []byte) (*domain.ExecutionProfile, error) {
var prof domain.ExecutionProfile
decoder := yaml.NewDecoder(bytes.NewReader(data))
decoder.KnownFields(true)
if err := decoder.Decode(&prof); err != nil {
return nil, err
}
if err := requireYAMLStreamEnd(decoder); err != nil {
return nil, err
}
return &prof, nil
}
func requireYAMLStreamEnd(decoder *yaml.Decoder) error {
var trailing yaml.Node
err := decoder.Decode(&trailing)
if errors.Is(err, io.EOF) {
return nil
}
if err != nil {
return err
}
return errors.New("profile file must contain exactly one YAML document")
}

50
internal/profile/memory_repository.go
View File

@@ -1,50 +0,0 @@
package profile
import (
"context"
"fmt"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
)
// NewMemoryRepository constructs an immutable repository from profile
// definitions supplied by an internal caller.
func NewMemoryRepository(definitions []domain.ExecutionProfile) (Repository, error) {
repository := &memoryRepository{profiles: make(map[string]domain.ExecutionProfile, len(definitions))}
for _, definition := range definitions {
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
if err != nil {
return nil, err
}
definition.ExtraParams = extraParams
if err := NormalizeAndValidateDefinition(&definition); err != nil {
return nil, err
}
if _, exists := repository.profiles[definition.ID]; exists {
return nil, fmt.Errorf("duplicate profile id %q", definition.ID)
}
repository.profiles[definition.ID] = definition
}
return repository, nil
}
type memoryRepository struct {
profiles map[string]domain.ExecutionProfile
}
func (r *memoryRepository) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) {
if r == nil {
return nil, ErrProfileNotFound
}
definition, found := r.profiles[id]
if !found {
return nil, ErrProfileNotFound
}
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
if err != nil {
return nil, fmt.Errorf("copy memory profile %q: %w", definition.ID, err)
}
definition.ExtraParams = extraParams
return &definition, nil
}

91
internal/profile/memory_repository_test.go
View File

@@ -1,91 +0,0 @@
package profile
import (
"context"
"errors"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
func TestMemoryRepositoryOwnsNormalizedDefinitions(t *testing.T) {
nested := map[string]any{"value": "original"}
definitions := []domain.ExecutionProfile{{
ID: " profile ",
BackendID: " backend ",
Endpoint: " https://example.test/v1/ ",
Model: "model",
ExtraParams: map[string]any{"nested": nested},
}}
repository, err := NewMemoryRepository(definitions)
if err != nil {
t.Fatalf("construct repository: %v", err)
}
nested["value"] = "mutated input"
first, err := repository.GetProfile(context.Background(), "profile")
if err != nil {
t.Fatalf("load profile: %v", err)
}
if first.ID != "profile" || first.BackendID != "backend" || first.Endpoint != "https://example.test/v1/" {
t.Fatalf("normalized definition = %#v", first)
}
if first.ExtraParams["nested"].(map[string]any)["value"] != "original" {
t.Fatalf("repository retained input mutation: %#v", first.ExtraParams)
}
first.ExtraParams["nested"].(map[string]any)["value"] = "mutated result"
second, err := repository.GetProfile(context.Background(), "profile")
if err != nil {
t.Fatalf("reload profile: %v", err)
}
if second.ExtraParams["nested"].(map[string]any)["value"] != "original" {
t.Fatalf("repository retained result mutation: %#v", second.ExtraParams)
}
}
func TestMemoryRepositoryPreservesInheritanceAndRejectsInvalidDefinitions(t *testing.T) {
derived, err := NewMemoryRepository([]domain.ExecutionProfile{{
ID: "derived",
BaseProfileID: "base",
ReasoningEffort: "high",
}})
if err != nil {
t.Fatalf("construct derived repository: %v", err)
}
profile, err := derived.GetProfile(context.Background(), "derived")
if err != nil {
t.Fatalf("load derived profile: %v", err)
}
if profile.BaseProfileID != "base" || profile.ReasoningEffort != "high" {
t.Fatalf("derived profile = %#v", profile)
}
if _, err := derived.GetProfile(context.Background(), "missing"); !errors.Is(err, ErrProfileNotFound) {
t.Fatalf("missing profile error = %v, want ErrProfileNotFound", err)
}
tests := []struct {
name string
definitions []domain.ExecutionProfile
}{
{
name: "normalized duplicate IDs",
definitions: []domain.ExecutionProfile{
{ID: " duplicate ", Endpoint: "https://one.example/v1", Model: "one"},
{ID: "duplicate", Endpoint: "https://two.example/v1", Model: "two"},
},
},
{
name: "invalid extra parameters",
definitions: []domain.ExecutionProfile{{ID: "invalid", Endpoint: "https://example.test/v1", Model: "model", ExtraParams: map[string]any{"invalid": func() {}}}},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
if _, err := NewMemoryRepository(test.definitions); err == nil {
t.Fatal("expected repository construction error")
}
})
}
}

6
internal/profile/repository.go
View File

@@ -2,11 +2,15 @@ package profile
import (
"context"
"errors"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
// Repository loads execution profiles.
// ErrProfileNotFound identifies an absent profile in the captured index.
var ErrProfileNotFound = errors.New("execution profile not found")
// Repository selects execution profiles from a captured index.
type Repository interface {
GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error)
}

58
internal/profile/repository_benchmark_test.go
View File

@@ -1,58 +0,0 @@
package profile
import (
"context"
"fmt"
"testing"
"testing/fstest"
)
func BenchmarkProfileRepositoryLookup(b *testing.B) {
for _, size := range []int{10, 1000} {
b.Run(fmt.Sprintf("catalog-%d", size), func(b *testing.B) {
files := fstest.MapFS{
"target.yaml": profileMapFile(`
id: target
endpoint: http://localhost:8000/v1
model: target-model
extra_params:
selected: true
`),
}
metadataNames := []string{"target.yaml"}
for i := 1; i < size; i++ {
name := fmt.Sprintf("profile-%04d.yaml", i)
files[name] = profileMapFile(fmt.Sprintf(`
id: profile-%04d
endpoint: http://localhost:8000/v1
model: unrelated-model
temperature: 0.5
max_tokens: 500
extra_params:
provider:
order:
- first
- second
`, i))
metadataNames = append(metadataNames, name)
}
fsys := &recordingProfileFS{FS: files}
repo := NewFSRepository(fsys, ".")
b.ReportAllocs()
b.ResetTimer()
for i := 0; i < b.N; i++ {
if _, err := repo.GetProfile(context.Background(), "target"); err != nil {
b.Fatal(err)
}
}
b.StopTimer()
for _, name := range metadataNames {
if got := fsys.openCount(name); got != b.N {
b.Fatalf("metadata %q opens = %d, want %d", name, got, b.N)
}
}
})
}
}

1140
internal/profile/repository_test.go
View File

File diff suppressed because it is too large Load Diff

160
internal/profile/resolving_repository.go
View File

@@ -1,160 +0,0 @@
package profile
import (
"context"
"errors"
"fmt"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
)
const maximumProfileChainLength = 32
type resolvingRepository struct {
source Repository
}
// NewResolvingRepository resolves inherited profile definitions from source.
func NewResolvingRepository(source Repository) Repository {
return &resolvingRepository{source: source}
}
func (r *resolvingRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
if r == nil || r.source == nil {
return nil, fmt.Errorf("%w: profile repository is required", ErrInvalidProfile)
}
requestedID := strings.TrimSpace(id)
if requestedID == "" {
return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile)
}
if err := ctx.Err(); err != nil {
return nil, err
}
profile, err := r.getRawProfile(ctx, requestedID)
if err != nil {
return nil, err
}
if profile == nil {
return nil, fmt.Errorf("%w: selected profile %q is nil", ErrInvalidProfile, requestedID)
}
chain := []*domain.ExecutionProfile{profile}
chainIDs := []string{requestedID}
visited := map[string]struct{}{requestedID: {}}
current := profile
for {
baseID := strings.TrimSpace(current.BaseProfileID)
if baseID == "" {
break
}
if err := ctx.Err(); err != nil {
return nil, err
}
if _, seen := visited[baseID]; seen {
return nil, fmt.Errorf("%w: profile inheritance cycle %s", ErrInvalidProfile, joinProfileChain(chainIDs, baseID))
}
if len(chain) >= maximumProfileChainLength {
return nil, fmt.Errorf("%w: profile inheritance chain exceeds %d profiles: %s", ErrInvalidProfile, maximumProfileChainLength, joinProfileChain(chainIDs, baseID))
}
base, err := r.getRawProfile(ctx, baseID)
if err != nil {
if errors.Is(err, ErrProfileNotFound) {
return nil, fmt.Errorf("%w: base profile %q is missing in chain %s", ErrInvalidProfile, baseID, joinProfileChain(chainIDs, baseID))
}
return nil, fmt.Errorf("%w: failed to load base profile %q in chain %s: %w", ErrInvalidProfile, baseID, joinProfileChain(chainIDs, baseID), err)
}
if base == nil {
return nil, fmt.Errorf("%w: base profile %q is nil in chain %s", ErrInvalidProfile, baseID, joinProfileChain(chainIDs, baseID))
}
chain = append(chain, base)
chainIDs = append(chainIDs, baseID)
visited[baseID] = struct{}{}
current = base
}
resolved, err := mergeProfileChain(chain)
if err != nil {
return nil, fmt.Errorf("%w: resolved profile chain %s: %w", ErrInvalidProfile, strings.Join(chainIDs, " -> "), err)
}
if err := validateResolvedProfile(resolved); err != nil {
return nil, fmt.Errorf("%w: resolved profile chain %s: %w", ErrInvalidProfile, strings.Join(chainIDs, " -> "), err)
}
return resolved, nil
}
func (r *resolvingRepository) getRawProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
profile, err := r.source.GetProfile(ctx, id)
if err != nil {
return nil, err
}
if err := ctx.Err(); err != nil {
return nil, err
}
return profile, nil
}
func joinProfileChain(chain []string, next string) string {
return strings.Join(append(append([]string(nil), chain...), next), " -> ")
}
func mergeProfileChain(chain []*domain.ExecutionProfile) (*domain.ExecutionProfile, error) {
resolved := &domain.ExecutionProfile{ID: chain[0].ID}
for index := len(chain) - 1; index >= 0; index-- {
definition := chain[index]
if strings.TrimSpace(definition.BackendID) != "" {
resolved.BackendID = definition.BackendID
}
if strings.TrimSpace(definition.Endpoint) != "" {
resolved.Endpoint = definition.Endpoint
}
if strings.TrimSpace(definition.Model) != "" {
resolved.Model = definition.Model
}
if definition.Temperature != 0 {
resolved.Temperature = definition.Temperature
}
if definition.MaxTokens != 0 {
resolved.MaxTokens = definition.MaxTokens
}
if definition.TopP != 0 {
resolved.TopP = definition.TopP
}
if definition.TimeoutSeconds != 0 {
resolved.TimeoutSeconds = definition.TimeoutSeconds
}
if strings.TrimSpace(definition.ServiceTier) != "" {
resolved.ServiceTier = definition.ServiceTier
}
if strings.TrimSpace(definition.ReasoningEffort) != "" {
resolved.ReasoningEffort = definition.ReasoningEffort
}
if strings.TrimSpace(definition.APIKeyEnv) != "" {
resolved.APIKeyEnv = definition.APIKeyEnv
}
resolved.APIKeyRequired = resolved.APIKeyRequired || definition.APIKeyRequired
if len(definition.ExtraParams) != 0 {
extraParams, err := jsonvalue.CopyMap(definition.ExtraParams)
if err != nil {
return nil, err
}
resolved.ExtraParams = extraParams
}
}
resolved.ID = chain[0].ID
resolved.BaseProfileID = ""
return resolved, nil
}
func validateResolvedProfile(profile *domain.ExecutionProfile) error {
if profile == nil {
return errors.New("resolved profile is required")
}
profile.BaseProfileID = ""
return NormalizeAndValidateDefinition(profile)
}

418
internal/profile/resolving_repository_test.go
View File

@@ -1,418 +0,0 @@
package profile
import (
"context"
"errors"
"fmt"
"io/fs"
"reflect"
"strings"
"sync"
"testing"
"testing/fstest"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
func TestResolvingRepositoryMergesProfileChain(t *testing.T) {
repo := &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
"leaf": {
ID: "leaf",
BaseProfileID: "middle",
BackendID: "leaf-backend",
TopP: 0.8,
TimeoutSeconds: 45,
ReasoningEffort: "high",
},
"middle": {
ID: "middle",
BaseProfileID: "root",
Endpoint: "https://middle.example/v1",
Model: "middle-model",
MaxTokens: 256,
APIKeyEnv: "MIDDLE_API_KEY",
APIKeyRequired: true,
ExtraParams: map[string]any{"middle": map[string]any{"value": "middle"}},
},
"root": {
ID: "root",
BackendID: "root-backend",
Endpoint: "https://root.example/v1",
Model: "root-model",
Temperature: 0.3,
ServiceTier: "priority",
ExtraParams: map[string]any{"root": "value"},
},
}}
got, err := NewResolvingRepository(repo).GetProfile(context.Background(), "leaf")
if err != nil {
t.Fatalf("resolve profile: %v", err)
}
want := &domain.ExecutionProfile{
ID: "leaf",
BackendID: "leaf-backend",
Endpoint: "https://middle.example/v1",
Model: "middle-model",
Temperature: 0.3,
MaxTokens: 256,
TopP: 0.8,
TimeoutSeconds: 45,
ServiceTier: "priority",
ReasoningEffort: "high",
APIKeyEnv: "MIDDLE_API_KEY",
APIKeyRequired: true,
ExtraParams: map[string]any{"middle": map[string]any{"value": "middle"}},
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("resolved profile:\n got %#v\nwant %#v", got, want)
}
}
func TestResolvingRepositoryRejectsMissingSourceAndProfileID(t *testing.T) {
if _, err := NewResolvingRepository(nil).GetProfile(context.Background(), "profile"); !errors.Is(err, ErrInvalidProfile) {
t.Fatalf("nil source error = %v, want ErrInvalidProfile", err)
}
repo := &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{}}
if _, err := NewResolvingRepository(repo).GetProfile(context.Background(), " \t "); !errors.Is(err, ErrInvalidProfile) {
t.Fatalf("blank id error = %v, want ErrInvalidProfile", err)
}
if got := repo.callCount(" "); got != 0 {
t.Fatalf("blank id looked up source %d times", got)
}
}
func TestResolvingRepositoryCopiesExtraParams(t *testing.T) {
baseParams := map[string]any{"nested": map[string]any{"value": "base"}}
repo := &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
"child": {ID: "child", BaseProfileID: "base"},
"base": {
ID: "base",
Endpoint: "https://base.example/v1",
Model: "model",
ExtraParams: baseParams,
},
}}
resolver := NewResolvingRepository(repo)
first, err := resolver.GetProfile(context.Background(), "child")
if err != nil {
t.Fatalf("resolve inherited map: %v", err)
}
first.ExtraParams["nested"].(map[string]any)["value"] = "mutated"
second, err := resolver.GetProfile(context.Background(), "child")
if err != nil {
t.Fatalf("resolve inherited map again: %v", err)
}
if got := second.ExtraParams["nested"].(map[string]any)["value"]; got != "base" {
t.Fatalf("later result retained mutation: %v", got)
}
if got := baseParams["nested"].(map[string]any)["value"]; got != "base" {
t.Fatalf("source map retained mutation: %v", got)
}
repo.set("child", &domain.ExecutionProfile{
ID: "child",
BaseProfileID: "base",
ExtraParams: map[string]any{"child": "replacement"},
})
replaced, err := resolver.GetProfile(context.Background(), "child")
if err != nil {
t.Fatalf("resolve replacement map: %v", err)
}
if !reflect.DeepEqual(replaced.ExtraParams, map[string]any{"child": "replacement"}) {
t.Fatalf("extra params = %#v, want complete child replacement", replaced.ExtraParams)
}
}
func TestResolvingRepositoryUsesRawOverlayForEachLookup(t *testing.T) {
leafSource := NewFSRepository(profileTestFS(map[string]string{
"leaf.yaml": "id: leaf\nbase_profile: base\n",
}), ".")
fallback := NewFSRepository(profileTestFS(map[string]string{
"base.yaml": "id: base\nendpoint: https://fallback.example/v1\nmodel: fallback-model\n",
}), ".")
overlay := NewOverlayRepository(leafSource, fallback)
resolver := NewResolvingRepository(overlay)
got, err := resolver.GetProfile(context.Background(), "leaf")
if err != nil {
t.Fatalf("resolve fallback base: %v", err)
}
if got.Model != "fallback-model" {
t.Fatalf("fallback base model = %q", got.Model)
}
shadowing := NewOverlayRepository(NewFSRepository(profileTestFS(map[string]string{
"leaf.yaml": "id: leaf\nbase_profile: base\n",
"base.yaml": "id: base\nendpoint: https://primary.example/v1\nmodel: primary-model\n",
}), "."), fallback)
got, err = NewResolvingRepository(shadowing).GetProfile(context.Background(), "leaf")
if err != nil {
t.Fatalf("resolve shadowed base: %v", err)
}
if got.Model != "primary-model" || got.Endpoint != "https://primary.example/v1" {
t.Fatalf("shadowed base = %+v", got)
}
}
func TestResolvingRepositoryReportsSafetyAndSourceErrors(t *testing.T) {
sourceErr := errors.New("source failure")
tests := []struct {
name string
repo *resolvingTestRepository
id string
want []error
wantNot error
contains []string
}{
{
name: "missing selected profile preserves not found",
repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{}},
id: "missing",
want: []error{ErrProfileNotFound},
wantNot: ErrInvalidProfile,
},
{
name: "missing base is invalid but not not found",
repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
"leaf": {ID: "leaf", BaseProfileID: "missing"},
}},
id: "leaf",
want: []error{ErrInvalidProfile},
wantNot: ErrProfileNotFound,
contains: []string{"missing", "leaf -> missing"},
},
{
name: "direct cycle",
repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
"a": {ID: "a", BaseProfileID: "a"},
}},
id: "a",
want: []error{ErrInvalidProfile},
contains: []string{"a -> a"},
},
{
name: "indirect cycle",
repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
"a": {ID: "a", BaseProfileID: "b"},
"b": {ID: "b", BaseProfileID: "c"},
"c": {ID: "c", BaseProfileID: "a"},
}},
id: "a",
want: []error{ErrInvalidProfile},
contains: []string{"a -> b -> c -> a"},
},
{
name: "nil result",
repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
"leaf": nil,
}},
id: "leaf",
want: []error{ErrInvalidProfile},
},
{
name: "incomplete resolved profile",
repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
"leaf": {ID: "leaf", BaseProfileID: "base"},
"base": {ID: "base", Model: "model"},
}},
id: "leaf",
want: []error{ErrInvalidProfile},
},
{
name: "base source error is retained",
repo: &resolvingTestRepository{
profiles: map[string]*domain.ExecutionProfile{"leaf": {ID: "leaf", BaseProfileID: "base"}},
errors: map[string]error{"base": sourceErr},
},
id: "leaf",
want: []error{ErrInvalidProfile, sourceErr},
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
_, err := NewResolvingRepository(tc.repo).GetProfile(context.Background(), tc.id)
for _, want := range tc.want {
if !errors.Is(err, want) {
t.Fatalf("error = %v, want %v", err, want)
}
}
if tc.wantNot != nil && errors.Is(err, tc.wantNot) {
t.Fatalf("error = %v, must not match %v", err, tc.wantNot)
}
for _, fragment := range tc.contains {
if !strings.Contains(err.Error(), fragment) {
t.Fatalf("error = %v, want %q", err, fragment)
}
}
})
}
}
func TestResolvingRepositoryEnforcesChainLength(t *testing.T) {
for _, count := range []int{maximumProfileChainLength, maximumProfileChainLength + 1} {
t.Run(fmt.Sprintf("%d profiles", count), func(t *testing.T) {
profiles := make(map[string]*domain.ExecutionProfile, count)
for index := 1; index <= count; index++ {
id := fmt.Sprintf("profile-%d", index)
definition := &domain.ExecutionProfile{ID: id}
if index == count {
definition.Endpoint = "https://root.example/v1"
definition.Model = "model"
} else {
definition.BaseProfileID = fmt.Sprintf("profile-%d", index+1)
}
profiles[id] = definition
}
got, err := NewResolvingRepository(&resolvingTestRepository{profiles: profiles}).GetProfile(context.Background(), "profile-1")
if count == maximumProfileChainLength {
if err != nil || got == nil {
t.Fatalf("profile = %+v, error = %v, want accepted chain", got, err)
}
return
}
if !errors.Is(err, ErrInvalidProfile) {
t.Fatalf("error = %v, want ErrInvalidProfile", err)
}
})
}
}
func TestResolvingRepositoryIsFreshAndCancellationAware(t *testing.T) {
repo := &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{
"leaf": {ID: "leaf", BaseProfileID: "base"},
"base": {ID: "base", Endpoint: "https://base.example/v1", Model: "first", ExtraParams: map[string]any{"nested": map[string]any{"value": "first"}}},
}}
resolver := NewResolvingRepository(repo)
first, err := resolver.GetProfile(context.Background(), "leaf")
if err != nil || first.Model != "first" {
t.Fatalf("first result=(%+v, %v)", first, err)
}
repo.set("base", &domain.ExecutionProfile{ID: "base", Endpoint: "https://base.example/v1", Model: "second", ExtraParams: map[string]any{"nested": map[string]any{"value": "second"}}})
second, err := resolver.GetProfile(context.Background(), "leaf")
if err != nil || second.Model != "second" {
t.Fatalf("second result=(%+v, %v)", second, err)
}
canceled, cancel := context.WithCancel(context.Background())
cancel()
if _, err := resolver.GetProfile(canceled, "leaf"); !errors.Is(err, context.Canceled) {
t.Fatalf("canceled lookup error = %v", err)
}
if got := repo.callCount("leaf"); got != 2 {
t.Fatalf("calls after canceled lookup = %d, want 2", got)
}
duringTraversal, cancelDuringTraversal := context.WithCancel(context.Background())
repo.afterGet = func(id string) {
if id == "leaf" {
cancelDuringTraversal()
}
}
if _, err := resolver.GetProfile(duringTraversal, "leaf"); !errors.Is(err, context.Canceled) {
t.Fatalf("during traversal error = %v", err)
}
if got := repo.callCount("base"); got != 2 {
t.Fatalf("base calls after cancellation = %d, want 2", got)
}
terminalLookup, cancelTerminalLookup := context.WithCancel(context.Background())
repo.afterGet = func(id string) {
if id == "base" {
cancelTerminalLookup()
}
}
if _, err := resolver.GetProfile(terminalLookup, "leaf"); !errors.Is(err, context.Canceled) {
t.Fatalf("terminal lookup cancellation error = %v", err)
}
if got := repo.callCount("base"); got != 3 {
t.Fatalf("base calls after terminal cancellation = %d, want 3", got)
}
repo.afterGet = nil
var wg sync.WaitGroup
errors := make(chan error, 8)
for index := 0; index < cap(errors); index++ {
wg.Add(1)
go func() {
defer wg.Done()
resolved, err := resolver.GetProfile(context.Background(), "leaf")
if err != nil {
errors <- err
return
}
resolved.ExtraParams["nested"].(map[string]any)["value"] = "mutated"
}()
}
wg.Wait()
close(errors)
for err := range errors {
t.Errorf("concurrent resolution: %v", err)
}
latest, err := resolver.GetProfile(context.Background(), "leaf")
if err != nil || latest.ExtraParams["nested"].(map[string]any)["value"] != "second" {
t.Fatalf("latest result=(%+v, %v)", latest, err)
}
}
type resolvingTestRepository struct {
mu sync.Mutex
profiles map[string]*domain.ExecutionProfile
errors map[string]error
calls map[string]int
afterGet func(string)
}
func (r *resolvingTestRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
if err := ctx.Err(); err != nil {
return nil, err
}
r.mu.Lock()
if r.calls == nil {
r.calls = make(map[string]int)
}
r.calls[id]++
err := r.errors[id]
profile := r.profiles[id]
afterGet := r.afterGet
r.mu.Unlock()
if afterGet != nil {
afterGet(id)
}
if err != nil {
return nil, err
}
if profile == nil {
if _, exists := r.profiles[id]; exists {
return nil, nil
}
return nil, ErrProfileNotFound
}
copy := *profile
return &copy, nil
}
func (r *resolvingTestRepository) set(id string, profile *domain.ExecutionProfile) {
r.mu.Lock()
defer r.mu.Unlock()
r.profiles[id] = profile
}
func (r *resolvingTestRepository) callCount(id string) int {
r.mu.Lock()
defer r.mu.Unlock()
return r.calls[id]
}
func profileTestFS(files map[string]string) fs.FS {
fsys := make(fstest.MapFS, len(files))
for name, content := range files {
fsys[name] = profileMapFile(content)
}
return fsys
}

3
internal/profile/testdata/invalid_yaml.yaml vendored
View File

@@ -1,3 +0,0 @@
id: invalid_yaml
endpoint: http://localhost:8000/v1
model: [broken

2
internal/profile/testdata/missing_endpoint.yaml vendored
View File

@@ -1,2 +0,0 @@
id: missing-endpoint
model: gpt-4o-mini

2
internal/profile/testdata/missing_id.yaml vendored
View File

@@ -1,2 +0,0 @@
endpoint: http://localhost:8000/v1
model: gpt-4o-mini

2
internal/profile/testdata/missing_model.yaml vendored
View File

@@ -1,2 +0,0 @@
id: missing-model
endpoint: http://localhost:8000/v1

4
internal/profile/testdata/raw_api_key.yaml vendored
View File

@@ -1,4 +0,0 @@
id: raw-api-key
endpoint: http://localhost:8000/v1
model: gpt-4o-mini
api_key: super-secret-should-not-be-here

4
internal/profile/testdata/unknown_field.yaml vendored
View File

@@ -1,4 +0,0 @@
id: unknown-field
endpoint: http://localhost:8000/v1
model: gpt-4o-mini
foo: bar

7
internal/profile/testdata/valid_local_profile.yaml vendored
View File

@@ -1,7 +0,0 @@
id: local-default
endpoint: http://localhost:8000/v1
model: gpt-4o-mini
temperature: 0.2
max_tokens: 700
top_p: 1.0
timeout_seconds: 120

8
internal/profile/testdata/valid_with_api_key_env.yaml vendored
View File

@@ -1,8 +0,0 @@
id: local-secure
endpoint: http://localhost:8000/v1
model: gpt-4o-mini
api_key_env: PROMPTKIT_API_KEY
service_tier: priority
reasoning_effort: medium
extra_params:
provider: local

98
internal/promptdef/content_source.go
View File

@@ -1,98 +0,0 @@
package promptdef
import (
"fmt"
"io/fs"
"os"
"path"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
)
type contentSourceRoot interface {
readContentFile(sourcePath string, contentFile string) (string, string, error)
}
type osContentSourceRoot struct {
root string
sourcePathsRelative bool
}
func (r osContentSourceRoot) readContentFile(sourcePath string, contentFile string) (string, string, error) {
if strings.TrimSpace(contentFile) == "" {
return "", "", fmt.Errorf("path is required")
}
if filepath.IsAbs(contentFile) {
return "", "", fmt.Errorf("path %q must be relative", contentFile)
}
root, err := filepath.Abs(r.root)
if err != nil {
return "", "", fmt.Errorf("resolve source root %q: %w", r.root, err)
}
canonicalRoot, err := filepath.EvalSymlinks(root)
if err != nil {
return "", "", fmt.Errorf("resolve source root %q: %w", r.root, err)
}
promptPath := sourcePath
if r.sourcePathsRelative && !filepath.IsAbs(promptPath) {
promptPath = filepath.Join(root, filepath.FromSlash(promptPath))
} else {
promptPath, err = filepath.Abs(promptPath)
if err != nil {
return "", "", fmt.Errorf("resolve prompt source %q: %w", sourcePath, err)
}
}
resolvedPath := filepath.Clean(filepath.Join(filepath.Dir(promptPath), contentFile))
if !containsOSPath(root, resolvedPath) {
return "", "", fmt.Errorf("path %q escapes source root %q", contentFile, r.root)
}
canonicalPath, err := filepath.EvalSymlinks(resolvedPath)
if err != nil {
return "", "", err
}
if !containsOSPath(canonicalRoot, canonicalPath) {
return "", "", fmt.Errorf("path %q escapes source root %q", contentFile, r.root)
}
body, err := os.ReadFile(canonicalPath)
if err != nil {
return "", "", err
}
return string(body), resolvedPath, nil
}
type fsContentSourceRoot struct {
fsys fs.FS
root string
}
func (r fsContentSourceRoot) readContentFile(sourcePath string, contentFile string) (string, string, error) {
root := filecatalog.CleanFSRoot(r.root)
cleanSourcePath := path.Clean(sourcePath)
if cleanSourcePath == root {
root = path.Dir(root)
}
resolvedPath, _, err := filecatalog.ResolveFSPath(root, path.Dir(cleanSourcePath), contentFile)
if err != nil {
return "", "", err
}
body, err := fs.ReadFile(r.fsys, resolvedPath)
if err != nil {
return "", "", err
}
return string(body), resolvedPath, nil
}
func containsOSPath(root string, name string) bool {
relative, err := filepath.Rel(root, name)
if err != nil || filepath.IsAbs(relative) {
return false
}
return relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator))
}

328
internal/promptdef/filesystem_repository.go
View File

@@ -1,328 +0,0 @@
package promptdef
import (
"bytes"
"context"
"errors"
"fmt"
"io"
"io/fs"
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gopkg.in/yaml.v3"
)
var (
ErrPromptDefinitionNotFound = errors.New("prompt definition not found")
ErrInvalidYAML = errors.New("invalid YAML format")
ErrInvalidPromptDefinition = errors.New("invalid prompt definition configuration")
)
type sourceRepository struct {
source promptDefinitionSource
}
type promptDefinitionFile struct {
ID string `yaml:"id"`
Version string `yaml:"version"`
DefaultProfile *string `yaml:"default_profile"`
Description string `yaml:"description"`
SessionID string `yaml:"session_id"`
Inputs []promptInputFile `yaml:"inputs"`
Messages []promptMessageFile `yaml:"messages"`
Output promptOutputContractFile `yaml:"output"`
}
type promptInputFile struct {
Name string `yaml:"name"`
Required bool `yaml:"required"`
ContentType string `yaml:"content_type"`
Description string `yaml:"description"`
}
type promptMessageFile struct {
Role string `yaml:"role"`
Content string `yaml:"content"`
ContentFile string `yaml:"content_file"`
CacheControl *cacheControlFile `yaml:"cache_control"`
}
type cacheControlFile struct {
Type string `yaml:"type"`
TTL string `yaml:"ttl"`
}
type promptOutputContractFile struct {
Format domain.OutputFormat `yaml:"format"`
ValidationMode domain.ValidationMode `yaml:"validation_mode"`
SchemaPath string `yaml:"schema_path"`
RepairAttempts int `yaml:"repair_attempts"`
}
func NewFilesystemRepository(dir string) Repository {
return &sourceRepository{
source: osPromptSource{
root: dir,
contentRoot: osContentSourceRoot{root: dir},
},
}
}
func NewFSRepository(fsys fs.FS, root string) Repository {
return &sourceRepository{
source: fsPromptSource{
fsys: fsys,
root: root,
contentRoot: fsContentSourceRoot{fsys: fsys, root: root},
},
}
}
// NewFileRepository constructs a repository for one operating-system prompt file.
func NewFileRepository(fsys fs.FS, file string, sourceDir string) Repository {
return &sourceRepository{
source: fsPromptSource{
fsys: fsys,
root: file,
contentRoot: osContentSourceRoot{
root: sourceDir,
sourcePathsRelative: true,
},
},
}
}
func (r *sourceRepository) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) {
if strings.TrimSpace(id) == "" {
return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidPromptDefinition)
}
if r == nil || r.source == nil {
return nil, errors.New("failed to read prompt definition directory: source is nil")
}
files, err := r.source.findYAMLFiles(ctx)
if err != nil {
return nil, fmt.Errorf("failed to read prompt definition directory: %w", err)
}
var matches []promptDefinitionMatch
for _, fullPath := range files {
select {
case <-ctx.Done():
return nil, ctx.Err()
default:
}
relPath := r.source.displayPath(fullPath)
data, err := r.source.readDefinition(fullPath)
if err != nil {
return nil, fmt.Errorf("failed to read prompt definition file %s: %w", relPath, err)
}
raw, err := decodePromptDefinition(data)
if err != nil {
if promptDefinitionDataMatches(data, id, version) {
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err)
}
continue
}
if !promptDefinitionMatches(raw, id, version) {
continue
}
matches = append(matches, promptDefinitionMatch{
raw: raw,
sourcePath: fullPath,
path: relPath,
})
}
if len(matches) > 1 {
paths := make([]string, 0, len(matches))
for _, match := range matches {
paths = append(paths, match.path)
}
if version != "" {
return nil, fmt.Errorf("%w: duplicate prompt definition id %q version %q found in: %s", ErrInvalidPromptDefinition, id, version, strings.Join(paths, ", "))
}
return nil, fmt.Errorf("%w: duplicate prompt definition id %q found in: %s", ErrInvalidPromptDefinition, id, strings.Join(paths, ", "))
}
if len(matches) == 1 {
match := matches[0]
def, err := normalizePromptDefinition(match.raw, r.source, match.sourcePath)
if err != nil {
return nil, fmt.Errorf("%w: %s: %v", ErrInvalidPromptDefinition, match.path, err)
}
return def, nil
}
return nil, ErrPromptDefinitionNotFound
}
type promptDefinitionMatch struct {
raw *promptDefinitionFile
sourcePath string
path string
}
func decodePromptDefinition(data []byte) (*promptDefinitionFile, error) {
var raw promptDefinitionFile
decoder := yaml.NewDecoder(bytes.NewReader(data))
decoder.KnownFields(true)
if err := decoder.Decode(&raw); err != nil {
return nil, err
}
var additional yaml.Node
if err := decoder.Decode(&additional); err != io.EOF {
if err != nil {
return nil, err
}
return nil, errors.New("prompt definition file must contain exactly one YAML document")
}
return &raw, nil
}
func promptDefinitionDataMatches(data []byte, id string, version string) bool {
var raw struct {
ID string `yaml:"id"`
Version string `yaml:"version"`
}
if err := yaml.NewDecoder(bytes.NewReader(data)).Decode(&raw); err != nil {
return false
}
return promptSelectorMatches(raw.ID, raw.Version, id, version)
}
func promptDefinitionMatches(raw *promptDefinitionFile, id string, version string) bool {
if raw == nil {
return false
}
return promptSelectorMatches(raw.ID, raw.Version, id, version)
}
func promptSelectorMatches(rawID string, rawVersion string, id string, version string) bool {
if strings.TrimSpace(rawID) != id {
return false
}
return version == "" || strings.TrimSpace(rawVersion) == version
}
func normalizePromptDefinition(raw *promptDefinitionFile, sourceRoot contentSourceRoot, sourcePath string) (*domain.PromptDefinition, error) {
return normalizePromptDefinitionWithContent(raw, func(contentFile string) (string, string, error) {
return sourceRoot.readContentFile(sourcePath, contentFile)
})
}
func normalizePromptDefinitionWithContent(raw *promptDefinitionFile, readContentFile func(string) (string, string, error)) (*domain.PromptDefinition, error) {
if raw == nil {
return nil, errors.New("prompt definition is nil")
}
id := strings.TrimSpace(raw.ID)
if id == "" {
return nil, errors.New("id is required")
}
version := strings.TrimSpace(raw.Version)
if version == "" {
return nil, errors.New("version is required")
}
if len(raw.Messages) == 0 {
return nil, errors.New("at least one message is required")
}
inputs := make([]domain.PromptInput, 0, len(raw.Inputs))
seenInputNames := make(map[string]struct{}, len(raw.Inputs))
for i, in := range raw.Inputs {
name := strings.TrimSpace(in.Name)
if name == "" {
return nil, fmt.Errorf("input %d has empty name", i)
}
if _, exists := seenInputNames[name]; exists {
return nil, fmt.Errorf("duplicate input name %q", name)
}
seenInputNames[name] = struct{}{}
inputs = append(inputs, domain.PromptInput{
Name: name,
Required: in.Required,
ContentType: strings.TrimSpace(in.ContentType),
Description: strings.TrimSpace(in.Description),
})
}
templates := make([]domain.PromptMessageTemplate, 0, len(raw.Messages))
for i, msg := range raw.Messages {
role, err := domain.NormalizeMessageRole(msg.Role)
if err != nil {
return nil, fmt.Errorf("message %d role: %w", i, err)
}
hasContent := strings.TrimSpace(msg.Content) != ""
hasContentFile := strings.TrimSpace(msg.ContentFile) != ""
if hasContent == hasContentFile {
return nil, fmt.Errorf("message %d must set exactly one of content or content_file", i)
}
var rawCacheControl *domain.CacheControl
if msg.CacheControl != nil {
rawCacheControl = &domain.CacheControl{
Type: domain.CacheControlType(msg.CacheControl.Type),
TTL: msg.CacheControl.TTL,
}
}
cacheControl, err := domain.NormalizeCacheControl(rawCacheControl)
if err != nil {
return nil, fmt.Errorf("message %d cache_control: %w", i, err)
}
templateContent := msg.Content
resolvedContentFile := ""
if hasContentFile {
body, resolvedPath, err := readContentFile(msg.ContentFile)
if err != nil {
return nil, fmt.Errorf("prompt %q message %d: failed to read content_file %q: %w", id, i, msg.ContentFile, err)
}
templateContent = body
resolvedContentFile = resolvedPath
}
templates = append(templates, domain.PromptMessageTemplate{
Role: role,
Content: templateContent,
ContentFile: resolvedContentFile,
CacheControl: cacheControl,
})
}
outputContract := domain.OutputContract{
Format: raw.Output.Format,
ValidationMode: raw.Output.ValidationMode,
SchemaPath: strings.TrimSpace(raw.Output.SchemaPath),
RepairAttempts: raw.Output.RepairAttempts,
}
if err := domain.ValidateOutputContract(outputContract); err != nil {
return nil, fmt.Errorf("output: %w", err)
}
defaultProfile := ""
if raw.DefaultProfile != nil {
defaultProfile = strings.TrimSpace(*raw.DefaultProfile)
if defaultProfile == "" {
return nil, errors.New("default_profile must be a non-empty string when set")
}
}
return &domain.PromptDefinition{
ID: id,
Version: version,
DefaultProfile: defaultProfile,
Description: strings.TrimSpace(raw.Description),
SessionID: strings.TrimSpace(raw.SessionID),
Inputs: inputs,
Templates: templates,
OutputFormat: raw.Output.Format,
Validation: outputContract,
}, nil
}

6
internal/promptdef/repository.go
View File

@@ -2,11 +2,15 @@ package promptdef
import (
"context"
"errors"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
)
// Repository loads prompt definitions.
// ErrPromptDefinitionNotFound identifies an absent captured prompt definition.
var ErrPromptDefinitionNotFound = errors.New("prompt definition not found")
// Repository selects prompt definitions from a captured index.
type Repository interface {
GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error)
}

1176
internal/promptdef/repository_test.go
View File

File diff suppressed because it is too large Load Diff

63
internal/promptdef/source.go
View File

@@ -1,63 +0,0 @@
package promptdef
import (
"context"
"errors"
"io/fs"
"os"
"gitea.maximumdirect.net/eric/promptkit/internal/filecatalog"
)
type promptDefinitionSource interface {
contentSourceRoot
findYAMLFiles(context.Context) ([]string, error)
readDefinition(string) ([]byte, error)
displayPath(string) string
}
type osPromptSource struct {
root string
contentRoot osContentSourceRoot
}
func (s osPromptSource) findYAMLFiles(ctx context.Context) ([]string, error) {
return filecatalog.FindYAMLFiles(ctx, s.root)
}
func (s osPromptSource) readDefinition(name string) ([]byte, error) {
return os.ReadFile(name)
}
func (s osPromptSource) displayPath(name string) string {
return filecatalog.RelativePath(s.root, name)
}
func (s osPromptSource) readContentFile(sourcePath string, contentFile string) (string, string, error) {
return s.contentRoot.readContentFile(sourcePath, contentFile)
}
type fsPromptSource struct {
fsys fs.FS
root string
contentRoot contentSourceRoot
}
func (s fsPromptSource) findYAMLFiles(ctx context.Context) ([]string, error) {
if s.fsys == nil {
return nil, errors.New("filesystem is nil")
}
return filecatalog.FindFSYAMLFiles(ctx, s.fsys, s.root)
}
func (s fsPromptSource) readDefinition(name string) ([]byte, error) {
return fs.ReadFile(s.fsys, name)
}
func (s fsPromptSource) displayPath(name string) string {
return filecatalog.DisplayPath(s.root, name)
}
func (s fsPromptSource) readContentFile(sourcePath string, contentFile string) (string, string, error) {
return s.contentRoot.readContentFile(sourcePath, contentFile)
}

10
internal/promptdef/testdata/both_content_and_content_file.yaml vendored
View File

@@ -1,10 +0,0 @@
id: both-content-and-content-file
version: "1.0.0"
messages:
- role: user
content: "Hi"
content_file: ./messages/user_prompt.tmpl
output:
format: text
validation_mode: none
repair_attempts: 0

14
internal/promptdef/testdata/duplicate_input_names.yaml vendored
View File

@@ -1,14 +0,0 @@
id: duplicate-input-names
version: "1.0.0"
inputs:
- name: transcript
required: true
- name: transcript
required: false
messages:
- role: user
content: "Hi"
output:
format: text
validation_mode: none
repair_attempts: 0

10
internal/promptdef/testdata/empty_cache_control_type.yaml vendored
View File

@@ -1,10 +0,0 @@
id: empty-cache-control-type
version: "1.0.0"
messages:
- role: system
content: "Use cached instructions."
cache_control: {}
output:
format: markdown
validation_mode: basic
repair_attempts: 0

9
internal/promptdef/testdata/invalid_validation_mode.yaml vendored
View File

@@ -1,9 +0,0 @@
id: invalid-validation-mode
version: "1.0.0"
messages:
- role: user
content: "Hi"
output:
format: text
validation_mode: nope
repair_attempts: 0

9
internal/promptdef/testdata/invalid_yaml.yaml vendored
View File

@@ -1,9 +0,0 @@
id: invalid-yaml
version: "1.0.0"
messages:
- role: user
content: [broken
output:
format: text
validation_mode: none
repair_attempts: 0

9
internal/promptdef/testdata/json_schema_without_schema_path.yaml vendored
View File

@@ -1,9 +0,0 @@
id: json-schema-without-schema-path
version: "1.0.0"
messages:
- role: user
content: "Return JSON"
output:
format: json
validation_mode: json_schema
repair_attempts: 0

2
internal/promptdef/testdata/messages/user_prompt.tmpl vendored
View File

@@ -1,2 +0,0 @@
Use transcript:
{{input "transcript"}}

9
internal/promptdef/testdata/missing_content_file.yaml vendored
View File

@@ -1,9 +0,0 @@
id: missing-content-file
version: "1.0.0"
messages:
- role: user
content_file: ./messages/does_not_exist.tmpl
output:
format: text
validation_mode: none
repair_attempts: 0

8
internal/promptdef/testdata/missing_id.yaml vendored
View File

@@ -1,8 +0,0 @@
version: "1.0.0"
messages:
- role: user
content: "Hi"
output:
format: text
validation_mode: none
repair_attempts: 0

8
internal/promptdef/testdata/neither_content_nor_content_file.yaml vendored
View File

@@ -1,8 +0,0 @@
id: neither-content-nor-content-file
version: "1.0.0"
messages:
- role: user
output:
format: text
validation_mode: none
repair_attempts: 0

6
internal/promptdef/testdata/no_messages.yaml vendored
View File

@@ -1,6 +0,0 @@
id: no-messages
version: "1.0.0"
output:
format: text
validation_mode: none
repair_attempts: 0

12
internal/promptdef/testdata/unknown_cache_control_field.yaml vendored
View File

@@ -1,12 +0,0 @@
id: unknown-cache-control-field
version: "1.0.0"
messages:
- role: system
content: "Use cached instructions."
cache_control:
type: ephemeral
unexpected: true
output:
format: markdown
validation_mode: basic
repair_attempts: 0

13
internal/promptdef/testdata/unknown_input_field.yaml vendored
View File

@@ -1,13 +0,0 @@
id: unknown-input-field
version: "1.0.0"
inputs:
- name: transcript
required: true
unknown_input_setting: true
messages:
- role: user
content: "Hi"
output:
format: text
validation_mode: none
repair_attempts: 0

12
internal/promptdef/testdata/unsupported_cache_control_ttl.yaml vendored
View File

@@ -1,12 +0,0 @@
id: unsupported-cache-control-ttl
version: "1.0.0"
messages:
- role: system
content: "Use cached instructions."
cache_control:
type: ephemeral
ttl: 5m
output:
format: markdown
validation_mode: basic
repair_attempts: 0

11
internal/promptdef/testdata/unsupported_cache_control_type.yaml vendored
View File

@@ -1,11 +0,0 @@
id: unsupported-cache-control-type
version: "1.0.0"
messages:
- role: system
content: "Use cached instructions."
cache_control:
type: persistent
output:
format: markdown
validation_mode: basic
repair_attempts: 0

14
internal/promptdef/testdata/valid_cache_control_ttl.yaml vendored
View File

@@ -1,14 +0,0 @@
id: valid-cache-control-ttl
version: "1.0.0"
messages:
- role: system
content: "Use cached instructions."
cache_control:
type: ephemeral
ttl: 1h
- role: user
content: "Summarize the input."
output:
format: markdown
validation_mode: basic
repair_attempts: 0

13
internal/promptdef/testdata/valid_cache_control_without_ttl.yaml vendored
View File

@@ -1,13 +0,0 @@
id: valid-cache-control-without-ttl
version: "1.0.0"
messages:
- role: system
content: "Use cached instructions."
cache_control:
type: ephemeral
- role: user
content: "Summarize the input."
output:
format: markdown
validation_mode: basic
repair_attempts: 0

14
internal/promptdef/testdata/valid_file_backed.yaml vendored
View File

@@ -1,14 +0,0 @@
id: valid-file-backed
version: "1.0.0"
inputs:
- name: transcript
required: true
messages:
- role: system
content: "Return markdown."
- role: user
content_file: ./messages/user_prompt.tmpl
output:
format: markdown
validation_mode: basic
repair_attempts: 0

18
internal/promptdef/testdata/valid_inline.yaml vendored
View File

@@ -1,18 +0,0 @@
id: valid-inline
version: "1.0.0"
inputs:
- name: transcript
required: true
content_type: text/markdown
description: Transcript content
messages:
- role: system
content: "You are concise."
- role: user
content: |
Summarize:
{{input "transcript"}}
output:
format: markdown
validation_mode: basic
repair_attempts: 0

10
internal/promptdef/testdata/valid_session_id.yaml vendored
View File

@@ -1,10 +0,0 @@
id: valid-session-id
version: "1.0.0"
session_id: " {{ .session_id }} "
messages:
- role: user
content: Hello.
output:
format: markdown
validation_mode: basic
repair_attempts: 0

13
internal/promptdef/testdata/with_default_profile.yaml vendored
View File

@@ -1,13 +0,0 @@
id: with-default-profile
version: "1.0.0"
default_profile: local-default
inputs:
- name: transcript
required: true
messages:
- role: user
content: "Write output"
output:
format: text
validation_mode: none
repair_attempts: 0

4
internal/usecase/execution_settings_test.go
View File

@@ -13,7 +13,7 @@ func TestRunnerPrepareExecutionRejectsInvalidExecutionSettings(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{forbid: true},
@@ -37,7 +37,7 @@ func TestRunnerPrepareExecutionValidatesAndNormalizesRequestEndpoints(t *testing
return NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{forbid: true},

2
internal/usecase/output_contract_test.go
View File

@@ -27,7 +27,7 @@ func newOutputContractTestRunner() (*Runner, outputContractTestCollaborators) {
return NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
collaborators.artifacts,
collaborators.renderer,
collaborators.llm,

26
internal/usecase/prepared_execution_test.go
View File

@@ -104,13 +104,13 @@ func TestRunnerPrepareExecutionCompletesWithoutAdmissionOrGeneration(t *testing.
}
admitter := &fakeRunAdmitter{}
profile := defaultExecutionProfile()
profile.ExtraParams = map[string]any{
profile.ResolvedTarget.ExtraParams = map[string]any{
"metadata": map[string]any{"source": "original"},
}
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
nil,
reader,
renderer,
llmClient,
@@ -210,7 +210,7 @@ func TestRunnerPreparationRejectsExcessivelyDeepPreparedSchema(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
defaultArtifactReader(),
defaultRenderer(),
llmClient,
@@ -265,9 +265,9 @@ func TestRunnerRunPreparedCredentialAvailabilityBeforeAdmission(t *testing.T) {
t.Setenv(environmentName, "available-during-preparation")
profile := defaultExecutionProfile()
profile.APIKeyRequired = tc.apiKeyRequired
profile.ResolvedTarget.APIKeyRequired = tc.apiKeyRequired
if tc.profileEnv {
profile.APIKeyEnv = environmentName
profile.ResolvedTarget.APIKeyEnv = environmentName
}
validator := &recordingValidationPreparer{plan: &recordingPreparedValidation{}}
admitter := &fakeRunAdmitter{}
@@ -275,7 +275,7 @@ func TestRunnerRunPreparedCredentialAvailabilityBeforeAdmission(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
nil,
defaultArtifactReader(),
defaultRenderer(),
llmClient,
@@ -319,12 +319,12 @@ func TestRunnerRunPreparedKeepsDirectCredentialOutOfMetadata(t *testing.T) {
const directKey = "direct-prepared-test-key"
profile := defaultExecutionProfile()
profile.APIKeyRequired = true
profile.ResolvedTarget.APIKeyRequired = true
client := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}},
nil,
defaultArtifactReader(),
defaultRenderer(),
client,
@@ -397,7 +397,7 @@ func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *test
runner := NewRunnerWithRepairer(
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
reader,
renderer,
client,
@@ -542,7 +542,7 @@ func TestRunnerRunPreparedReleasesAdmissionAcrossExecutionErrors(t *testing.T) {
runner := NewRunnerWithRepairer(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{
@@ -585,7 +585,7 @@ func TestRunnerPreparedExecutionOwnershipUseAndDiscard(t *testing.T) {
return NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
@@ -635,7 +635,7 @@ func TestRunnerPreparedExecutionWithoutValidatorSkipsValidation(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{resp: &domain.GenerateResponse{Content: `{}`}},
@@ -665,7 +665,7 @@ func TestRunnerPrepareExecutionRequiresValidationPreparer(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
reader,
defaultRenderer(),
&fakeLLM{forbid: true},

37
internal/usecase/profile_inspection.go
View File

@@ -7,12 +7,12 @@ import (
"strings"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue"
)
type resolvedProfileSelection struct {
id string
profile *domain.ExecutionProfile
backend *domain.Backend
}
func (r *Runner) resolveProfileSelection(
@@ -26,10 +26,6 @@ func (r *Runner) resolveProfileSelection(
if r == nil || r.profiles == nil {
return nil, fmt.Errorf("%w: profile repository is not configured", ErrProfileLoad)
}
if exact, ok := r.profiles.(interface{ ExactIDs() bool }); !ok || !exact.ExactIDs() {
lookupID = strings.TrimSpace(lookupID)
}
selectedProfile, err := r.profiles.GetProfile(ctx, lookupID)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
@@ -39,27 +35,13 @@ func (r *Runner) resolveProfileSelection(
}
profileValue := *selectedProfile
if profileValue.ResolvedTarget != nil {
return &resolvedProfileSelection{id: lookupID, profile: &profileValue}, nil
}
profileValue.BackendID = strings.TrimSpace(profileValue.BackendID)
var selectedBackend *domain.Backend
if profileValue.BackendID != "" {
if r.backends == nil {
return nil, fmt.Errorf("%w: backend %q cannot be resolved", ErrProfileLoad, profileValue.BackendID)
}
backendValue, err := r.backends.GetBackend(profileValue.BackendID)
if err != nil {
return nil, fmt.Errorf("%w: backend %q: %w", ErrProfileLoad, profileValue.BackendID, err)
}
selectedBackend = &backendValue
if profileValue.ResolvedTarget == nil {
return nil, fmt.Errorf("%w: profile %q has no resolved target", ErrProfileLoad, lookupID)
}
return &resolvedProfileSelection{
id: lookupID,
profile: &profileValue,
backend: selectedBackend,
}, nil
}
@@ -96,14 +78,13 @@ func (r *Runner) InspectProfile(
if err != nil {
return nil, err
}
target, _ := resolveExecutionTarget(selection.backend, selection.profile, nil)
if selection.profile.ResolvedTarget == nil {
target, err = normalizeResolvedExecutionTarget(target)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
}
}
target, _ := resolveExecutionTarget(selection.profile, nil)
target.APIKey = ""
params, err := jsonvalue.CopyMap(target.ExtraParams)
if err != nil {
return nil, fmt.Errorf("%w: profile extra parameters: %w", ErrProfileLoad, err)
}
target.ExtraParams = params
return &domain.ProfileInspection{
ProfileID: selection.id,

215
internal/usecase/profile_inspection_test.go
View File

@@ -6,7 +6,6 @@ import (
"reflect"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
)
@@ -18,10 +17,7 @@ type inspectionProfileRepository struct {
id string
}
func (r *inspectionProfileRepository) GetProfile(
_ context.Context,
id string,
) (*domain.ExecutionProfile, error) {
func (r *inspectionProfileRepository) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) {
r.calls++
r.id = id
if r.err != nil {
@@ -30,184 +26,65 @@ func (r *inspectionProfileRepository) GetProfile(
return r.profile, nil
}
type inspectionBackendResolver struct {
backend domain.Backend
err error
calls int
id string
}
func (r *inspectionBackendResolver) GetBackend(id string) (domain.Backend, error) {
r.calls++
r.id = id
if r.err != nil {
return domain.Backend{}, r.err
func TestRunnerInspectProfileUsesExactSnapshotAndCopiesTarget(t *testing.T) {
configured := domain.ExecutionTarget{
BackendID: "backend", Endpoint: "https://backend.example/v1", Model: "model",
Temperature: 0, APIKeyEnv: "OPTIONAL_KEY", ExtraParams: map[string]any{"nested": map[string]any{"value": "original"}},
}
return r.backend, nil
}
func TestRunnerInspectProfileResolvesProfileAndBackendOnce(t *testing.T) {
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
ID: "profile",
BackendID: " backend ",
Model: "profile-model",
Temperature: 0.4,
MaxTokens: 32,
TimeoutSeconds: 45,
ServiceTier: "priority",
ReasoningEffort: "high",
ExtraParams: map[string]any{
"profile": "value",
},
ID: " literal ", ResolvedTarget: &configured,
TargetPresence: domain.ExecutionTargetPresence{Temperature: true},
}}
backends := &inspectionBackendResolver{backend: domain.Backend{
ID: "backend",
Endpoint: "https://backend.example/v1",
APIKeyEnv: "BACKEND_KEY",
ExtraParams: map[string]any{
"backend": "value",
},
}}
runner := &Runner{profiles: profiles, backends: backends}
inspection, err := runner.InspectProfile(context.Background(), " profile ")
runner := &Runner{profiles: profiles}
first, err := runner.InspectProfile(context.Background(), " literal ")
if err != nil {
t.Fatalf("inspect profile: %v", err)
t.Fatalf("inspect exact ID: %v", err)
}
if profiles.calls != 1 || profiles.id != "profile" {
t.Fatalf("profile lookup=(calls=%d id=%q), want one exact lookup", profiles.calls, profiles.id)
if profiles.calls != 1 || profiles.id != " literal " || first.ProfileID != profiles.id || !reflect.DeepEqual(first.EffectiveModelParams, configured) {
t.Fatalf("inspection=%+v lookup=(%d,%q), want exact snapshot", first, profiles.calls, profiles.id)
}
if backends.calls != 1 || backends.id != "backend" {
t.Fatalf("backend lookup=(calls=%d id=%q), want one exact lookup", backends.calls, backends.id)
}
if profiles.profile.BackendID != " backend " {
t.Fatalf("inspection mutated repository profile backend: %q", profiles.profile.BackendID)
}
wantTarget := domain.ExecutionTarget{
BackendID: "backend",
Endpoint: "https://backend.example/v1",
Model: "profile-model",
Temperature: 0.4,
MaxTokens: 32,
TopP: defaults.ExecutionTargetDefault().TopP,
TimeoutSeconds: 45,
ServiceTier: "priority",
ReasoningEffort: "high",
APIKeyEnv: "BACKEND_KEY",
ExtraParams: map[string]any{
"profile": "value",
},
}
if inspection.ProfileID != "profile" || inspection.APIKeyRequired ||
!reflect.DeepEqual(inspection.EffectiveModelParams, wantTarget) {
t.Fatalf("inspection=%#v, want profile=%q target=%#v", inspection, "profile", wantTarget)
first.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["value"] = "changed"
second, err := runner.InspectProfile(context.Background(), " literal ")
if err != nil || second.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["value"] != "original" {
t.Fatalf("inspection mutated snapshot: (%+v, %v)", second, err)
}
}
func TestRunnerInspectProfileDoesNotNeedExecutionCollaboratorsOrCredentials(t *testing.T) {
t.Setenv("PROMPTKIT_INSPECTION_TEST_KEY", "")
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
ID: "endpoint-only",
Endpoint: "https://profile.example/v1",
Model: "profile-model",
APIKeyEnv: "PROMPTKIT_INSPECTION_TEST_KEY",
ID: "endpoint-only", ResolvedTarget: &domain.ExecutionTarget{
Endpoint: "https://profile.example/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_INSPECTION_TEST_KEY",
},
}}
inspection, err := (&Runner{profiles: profiles}).InspectProfile(context.Background(), "endpoint-only")
if err != nil || inspection.EffectiveModelParams.APIKeyEnv != "PROMPTKIT_INSPECTION_TEST_KEY" || inspection.APIKeyRequired {
t.Fatalf("unexpected endpoint-only inspection: (%+v, %v)", inspection, err)
}
}
func TestRunnerInspectProfileClassifiesLookupFailures(t *testing.T) {
profiles := &inspectionProfileRepository{}
runner := &Runner{profiles: profiles}
inspection, err := runner.InspectProfile(context.Background(), "endpoint-only")
if err != nil {
t.Fatalf("inspect endpoint-only profile: %v", err)
if _, err := runner.InspectProfile(context.Background(), " \t "); !errors.Is(err, ErrInvalidRequest) || profiles.calls != 0 {
t.Fatalf("blank inspection=(%v, calls=%d)", err, profiles.calls)
}
if inspection.EffectiveModelParams.BackendID != "" ||
inspection.EffectiveModelParams.APIKeyEnv != "PROMPTKIT_INSPECTION_TEST_KEY" ||
inspection.APIKeyRequired {
t.Fatalf("unexpected endpoint-only inspection: %#v", inspection)
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := runner.InspectProfile(ctx, "profile"); !errors.Is(err, ErrProfileLoad) || !errors.Is(err, context.Canceled) || profiles.calls != 0 {
t.Fatalf("canceled inspection=(%v, calls=%d)", err, profiles.calls)
}
profiles.err = profile.ErrProfileNotFound
if _, err := runner.InspectProfile(context.Background(), "missing"); !errors.Is(err, ErrProfileLoad) || !errors.Is(err, profile.ErrProfileNotFound) {
t.Fatalf("missing profile error=%v", err)
}
for _, invalid := range []*Runner{
{},
{profiles: &inspectionProfileRepository{}},
{profiles: &inspectionProfileRepository{profile: &domain.ExecutionProfile{ID: "profile"}}},
} {
if _, err := invalid.InspectProfile(context.Background(), "profile"); !errors.Is(err, ErrProfileLoad) {
t.Fatalf("invalid dependency error=%v", err)
}
}
}
func TestRunnerInspectProfileDirectCredentialRequirementClearsBackendEnvironment(t *testing.T) {
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
ID: "direct-key",
BackendID: "backend",
Model: "profile-model",
APIKeyRequired: true,
}}
backends := &inspectionBackendResolver{backend: domain.Backend{
ID: "backend",
Endpoint: "https://backend.example/v1",
APIKeyEnv: "BACKEND_KEY",
}}
inspection, err := (&Runner{profiles: profiles, backends: backends}).InspectProfile(
context.Background(),
"direct-key",
)
if err != nil {
t.Fatalf("inspect direct-key profile: %v", err)
}
if !inspection.APIKeyRequired || inspection.EffectiveModelParams.APIKeyEnv != "" {
t.Fatalf("credential requirement was not resolved exclusively: %#v", inspection)
}
}
func TestRunnerInspectProfileClassifiesFailuresWithoutRepositoryWorkAfterCancellation(t *testing.T) {
t.Run("blank ID", func(t *testing.T) {
profiles := &inspectionProfileRepository{}
_, err := (&Runner{profiles: profiles}).InspectProfile(context.Background(), " \t ")
if !errors.Is(err, ErrInvalidRequest) || profiles.calls != 0 {
t.Fatalf("blank inspection=(%v, calls=%d), want invalid request without lookup", err, profiles.calls)
}
})
t.Run("canceled context", func(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
profiles := &inspectionProfileRepository{}
_, err := (&Runner{profiles: profiles}).InspectProfile(ctx, "profile")
if !errors.Is(err, ErrProfileLoad) || !errors.Is(err, context.Canceled) || profiles.calls != 0 {
t.Fatalf("canceled inspection=(%v, calls=%d), want profile load and context identities without lookup", err, profiles.calls)
}
})
t.Run("missing profile", func(t *testing.T) {
profiles := &inspectionProfileRepository{err: profile.ErrProfileNotFound}
_, err := (&Runner{profiles: profiles}).InspectProfile(context.Background(), "missing")
if !errors.Is(err, ErrProfileLoad) || !errors.Is(err, profile.ErrProfileNotFound) {
t.Fatalf("missing profile error=%v, want profile load and not-found identities", err)
}
})
t.Run("unknown backend", func(t *testing.T) {
backendErr := errors.New("unknown backend")
profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{
ID: "profile", BackendID: "backend", Model: "profile-model",
}}
backends := &inspectionBackendResolver{err: backendErr}
_, err := (&Runner{profiles: profiles, backends: backends}).InspectProfile(context.Background(), "profile")
if !errors.Is(err, ErrProfileLoad) || !errors.Is(err, backendErr) {
t.Fatalf("unknown backend error=%v, want profile load and backend identities", err)
}
})
t.Run("defensive invalid dependencies", func(t *testing.T) {
cases := []struct {
name string
runner *Runner
}{
{name: "nil repository", runner: &Runner{}},
{name: "nil profile", runner: &Runner{profiles: &inspectionProfileRepository{}}},
{name: "invalid target", runner: &Runner{profiles: &inspectionProfileRepository{
profile: &domain.ExecutionProfile{ID: "profile", Endpoint: "https://profile.example/v1"},
}}},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
_, err := tc.runner.InspectProfile(context.Background(), "profile")
if !errors.Is(err, ErrProfileLoad) {
t.Fatalf("inspection error=%v, want profile load", err)
}
})
}
})
}

2
internal/usecase/prompt_inspection_test.go
View File

@@ -135,7 +135,7 @@ func TestRunnerPrepareUsesThePromptInspectionSelectionAndHash(t *testing.T) {
runner := NewRunner(
repository,
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
&fakeArtifactReader{},
&fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hello"}}}},
nil,

110
internal/usecase/runner.go
View File

@@ -44,7 +44,6 @@ var (
type Runner struct {
promptDefs promptdef.Repository
profiles profile.Repository
backends BackendResolver
artifacts artifact.Reader
renderer prompt.Renderer
llm llm.Client
@@ -54,11 +53,6 @@ type Runner struct {
defaultTransientRetries uint64
}
// BackendResolver resolves one normalized backend ID.
type BackendResolver interface {
GetBackend(string) (domain.Backend, error)
}
// RunAdmitter reserves capacity for one resolved backend run.
type RunAdmitter interface {
Admit(context.Context, string) (func(), error)
@@ -84,7 +78,6 @@ type preparedOperation struct {
func NewRunner(
promptDefs promptdef.Repository,
profiles profile.Repository,
backends BackendResolver,
artifacts artifact.Reader,
renderer prompt.Renderer,
llmClient llm.Client,
@@ -94,7 +87,6 @@ func NewRunner(
return NewRunnerWithRepairer(
promptDefs,
profiles,
backends,
artifacts,
renderer,
llmClient,
@@ -107,7 +99,6 @@ func NewRunner(
func NewRunnerWithRepairer(
promptDefs promptdef.Repository,
profiles profile.Repository,
backends BackendResolver,
artifacts artifact.Reader,
renderer prompt.Renderer,
llmClient llm.Client,
@@ -123,7 +114,6 @@ func NewRunnerWithRepairer(
return &Runner{
promptDefs: promptDefs,
profiles: profiles,
backends: backends,
artifacts: artifacts,
renderer: renderer,
llm: llmClient,
@@ -355,7 +345,7 @@ func (r *Runner) resolvePreparation(
recorder.UpdateSelection(observation.Selection{PromptID: def.ID, PromptVersion: def.Version, ProfileID: selection.id})
}
effectiveModel, targetPresence := resolveExecutionTarget(selection.backend, selection.profile, req.Execution)
effectiveModel, targetPresence := resolveExecutionTarget(selection.profile, req.Execution)
effectiveModel.APIKey = req.APIKey
effectiveModel, err = normalizeResolvedExecutionTarget(effectiveModel)
if err != nil {
@@ -578,48 +568,6 @@ func (r *Runner) shouldAttemptRepair(contract domain.OutputContract, validationR
}
}
func mergeExecutionTarget(base domain.ExecutionTarget, override domain.ExecutionTarget) domain.ExecutionTarget {
out := base
if strings.TrimSpace(override.BackendID) != "" {
out.BackendID = override.BackendID
}
if strings.TrimSpace(override.Endpoint) != "" {
out.Endpoint = override.Endpoint
}
if override.Model != "" {
out.Model = override.Model
}
if override.Temperature != 0 {
out.Temperature = override.Temperature
}
if override.MaxTokens != 0 {
out.MaxTokens = override.MaxTokens
}
if override.TopP != 0 {
out.TopP = override.TopP
}
if override.TimeoutSeconds != 0 {
out.TimeoutSeconds = override.TimeoutSeconds
}
if strings.TrimSpace(override.ServiceTier) != "" {
out.ServiceTier = override.ServiceTier
}
if strings.TrimSpace(override.ReasoningEffort) != "" {
out.ReasoningEffort = override.ReasoningEffort
}
if strings.TrimSpace(override.APIKeyEnv) != "" {
out.APIKeyEnv = override.APIKeyEnv
}
if override.APIKeyRequired {
out.APIKeyRequired = true
out.APIKeyEnv = ""
}
if len(override.ExtraParams) > 0 {
out.ExtraParams = copyExtraParams(override.ExtraParams)
}
return out
}
func mergeExecutionTargetOverride(base domain.ExecutionTarget, presence domain.ExecutionTargetPresence, override domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence) {
out := base
if override.Endpoint != "" {
@@ -659,22 +607,12 @@ func mergeExecutionTargetOverride(base domain.ExecutionTarget, presence domain.E
return out, presence
}
func resolveExecutionTarget(backendValue *domain.Backend, profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence) {
if profileValue != nil && profileValue.ResolvedTarget != nil {
out := *profileValue.ResolvedTarget
out.ExtraParams = copyExtraParams(out.ExtraParams)
presence := profileValue.TargetPresence
if override != nil {
return mergeExecutionTargetOverride(out, presence, *override)
}
return out, presence
}
out := defaults.ExecutionTargetDefault()
out = mergeExecutionTarget(out, backendToTarget(backendValue))
out = mergeExecutionTarget(out, executionProfileToTarget(profileValue))
var presence domain.ExecutionTargetPresence
func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence) {
out := *profileValue.ResolvedTarget
out.ExtraParams = copyExtraParams(out.ExtraParams)
presence := profileValue.TargetPresence
if override != nil {
out, presence = mergeExecutionTargetOverride(out, presence, *override)
return mergeExecutionTargetOverride(out, presence, *override)
}
return out, presence
}
@@ -696,42 +634,6 @@ func validateAPIKey(apiKeyEnv string, apiKey string, apiKeyRequired bool) error
return nil
}
func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget {
if p == nil {
return domain.ExecutionTarget{}
}
endpoint := p.Endpoint
if strings.TrimSpace(endpoint) == "" {
endpoint = ""
}
return domain.ExecutionTarget{
BackendID: p.BackendID,
Endpoint: endpoint,
Model: p.Model,
Temperature: p.Temperature,
MaxTokens: p.MaxTokens,
TopP: p.TopP,
TimeoutSeconds: p.TimeoutSeconds,
ServiceTier: p.ServiceTier,
ReasoningEffort: p.ReasoningEffort,
APIKeyEnv: p.APIKeyEnv,
APIKeyRequired: p.APIKeyRequired,
ExtraParams: copyExtraParams(p.ExtraParams),
}
}
func backendToTarget(value *domain.Backend) domain.ExecutionTarget {
if value == nil {
return domain.ExecutionTarget{}
}
return domain.ExecutionTarget{
BackendID: value.ID,
Endpoint: value.Endpoint,
APIKeyEnv: value.APIKeyEnv,
ExtraParams: copyExtraParams(value.ExtraParams),
}
}
func copyExtraParams(src map[string]any) map[string]any {
if len(src) == 0 {
return nil

559
internal/usecase/runner_test.go
View File

@@ -7,7 +7,6 @@ import (
"errors"
"fmt"
"math"
"path/filepath"
"reflect"
"regexp"
"strconv"
@@ -20,9 +19,7 @@ import (
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
"gitea.maximumdirect.net/eric/promptkit/internal/prompt"
"gitea.maximumdirect.net/eric/promptkit/internal/promptdef"
"gitea.maximumdirect.net/eric/promptkit/internal/validate"
)
@@ -39,10 +36,6 @@ type fakeExecutionProfileRepo struct {
lastID string
}
type fakeBackendResolver struct {
backends map[string]domain.Backend
}
type staticPromptRepo struct {
def *domain.PromptDefinition
}
@@ -89,14 +82,6 @@ func (staticRenderer) Render(
}, nil
}
func (f fakeBackendResolver) GetBackend(id string) (domain.Backend, error) {
value, ok := f.backends[id]
if !ok {
return domain.Backend{}, errors.New("backend not found")
}
return value, nil
}
func (f *fakeExecutionProfileRepo) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
f.lastID = id
if f.err != nil {
@@ -371,7 +356,7 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) {
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
llmClient := &fakeLLM{forbid: true}
runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil, nil)
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
PromptVersion: "1",
@@ -422,7 +407,7 @@ func TestRunnerDirectSessionResolution(t *testing.T) {
runner := NewRunner(
promptRepo,
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
defaultArtifactReader(),
prompt.NewGoRenderer(),
&fakeLLM{forbid: true},
@@ -469,7 +454,7 @@ func TestRunnerDirectSessionResolution(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
defaultArtifactReader(),
prompt.NewGoRenderer(),
&fakeLLM{forbid: true},
@@ -499,7 +484,7 @@ func TestRunnerDirectSessionResolution(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
defaultArtifactReader(),
prompt.NewGoRenderer(),
&fakeLLM{forbid: true},
@@ -526,7 +511,7 @@ func TestRunnerDirectSessionResolution(t *testing.T) {
runner := NewRunner(
promptRepo,
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
defaultArtifactReader(),
defaultRenderer(),
llmClient,
@@ -555,7 +540,7 @@ func TestRunnerDirectSessionResolution(t *testing.T) {
runner := NewRunner(
promptRepo,
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
defaultArtifactReader(),
defaultRenderer(),
llmClient,
@@ -583,7 +568,7 @@ func TestRunnerPrepareUsesPromptDefaultProfileWhenNoExplicitProfileID(t *testing
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
promptRepo.def.DefaultProfile = "from-prompt"
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"from-prompt": {ID: "from-prompt", Endpoint: "http://llm/v1", Model: "m"},
"from-prompt": {ID: "from-prompt", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m", TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds}},
}}
runner := newMinimalRunner(promptRepo, execRepo)
@@ -626,7 +611,7 @@ func TestRunnerPrepareSelectedProfileDoesNotExistFails(t *testing.T) {
}
func TestRunnerPreparePromptLoadFailure(t *testing.T) {
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil, nil)
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil, nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p"})
if !errors.Is(err, ErrPromptLoad) {
t.Fatalf("expected ErrPromptLoad, got %v", err)
@@ -640,17 +625,16 @@ func TestRunnerPrepareRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) {
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {
ID: "exec",
Endpoint: "http://profile/v1",
Model: "profile-model",
Temperature: 0.2,
MaxTokens: 500,
TopP: 0.9,
TimeoutSeconds: 120,
ServiceTier: "priority",
ID: "exec", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://profile/v1",
Model: "profile-model",
Temperature: 0.2,
MaxTokens: 500,
TopP: 0.9,
TimeoutSeconds: 120,
ServiceTier: "priority"},
},
}}
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil, nil)
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -740,15 +724,14 @@ func TestRunnerPrepareRequestNumericOverridePresence(t *testing.T) {
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {
ID: "exec",
Endpoint: "http://profile/v1",
Model: "profile-model",
Temperature: 0.7,
MaxTokens: 321,
TopP: 0.8,
TimeoutSeconds: 45,
ID: "exec", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://profile/v1",
Model: "profile-model",
Temperature: 0.7,
MaxTokens: 321,
TopP: 0.8,
TimeoutSeconds: 45},
},
}}, nil,
}},
defaultArtifactReader(),
defaultRenderer(),
@@ -802,7 +785,7 @@ func TestRunnerPrepareInvalidRequestNumericOverridesFail(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(),
defaultRenderer(),
@@ -826,15 +809,14 @@ func TestRunnerPrepareSelectedProfileBeatsBuiltInDefault(t *testing.T) {
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {
ID: "exec",
Endpoint: "http://profile/v1",
Model: "profile-model",
TopP: 0.8,
TimeoutSeconds: 90,
ServiceTier: "priority",
ID: "exec", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://profile/v1",
Model: "profile-model",
TopP: 0.8,
TimeoutSeconds: 90,
ServiceTier: "priority"},
},
}}
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil, nil)
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -855,54 +837,13 @@ func TestRunnerPrepareSelectedProfileBeatsBuiltInDefault(t *testing.T) {
}
}
func TestRunnerPrepareFileBackedPromptBodiesRenderCorrectly(t *testing.T) {
promptDir := filepath.Join("..", "promptdef", "testdata")
profileDir := filepath.Join("..", "profile", "testdata")
reader := &fakeArtifactReader{
artifactsByURI: map[string]*domain.Artifact{
"a://transcript": {
Name: "transcript",
Body: []byte("Session transcript body."),
Hash: hashString("Session transcript body."),
},
},
}
llmClient := &fakeLLM{forbid: true}
runner := NewRunner(
promptdef.NewFilesystemRepository(promptDir),
profile.NewFilesystemRepository(profileDir), nil,
reader,
prompt.NewGoRenderer(),
llmClient,
nil, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "valid-file-backed",
ProfileID: "local-default",
Inputs: map[string]domain.ArtifactRef{
"transcript": {Type: domain.ArtifactRefFile, URI: "a://transcript"},
},
})
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
if len(prepared.Messages) != 2 {
t.Fatalf("expected two rendered messages, got %d", len(prepared.Messages))
}
if !strings.Contains(prepared.Messages[1].Content, "Session transcript body.") {
t.Fatalf("expected file-backed template content to render input, got %q", prepared.Messages[1].Content)
}
}
func TestRunnerPrepareRequiredInputMissingFails(t *testing.T) {
def := promptDef(domain.FormatText, domain.ValidationNone, 0)
def.Templates = []domain.PromptMessageTemplate{{Role: "user", Content: `{{input "transcript"}}`}}
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(),
prompt.NewGoRenderer(),
@@ -929,7 +870,7 @@ func TestRunnerPrepareUnknownTemplateInputReferenceFails(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(),
prompt.NewGoRenderer(),
@@ -955,9 +896,9 @@ func TestRunnerPrepareAPIKeyEnvNameIncludedButNotResolvedValue(t *testing.T) {
t.Setenv(envName, secret)
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName},
"exec": {ID: "exec", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName, TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds}},
}}
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil, nil)
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if err != nil {
@@ -985,7 +926,7 @@ func TestRunnerPrepareJSONSchemaBuildsStructuredOutputSpec(t *testing.T) {
}
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(),
defaultRenderer(),
@@ -1029,7 +970,7 @@ func TestRunnerPrepareJSONSchemaSchemaLoadFailureReturnsValidationError(t *testi
validator := &fakeValidator{schemaErr: errors.New("schema unavailable")}
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(),
defaultRenderer(),
@@ -1059,7 +1000,7 @@ func TestRunnerRunJSONSchemaSchemaLoadFailureFailsBeforeLLM(t *testing.T) {
validator := &fakeValidator{schemaErr: errors.New("schema unavailable")}
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(),
defaultRenderer(),
@@ -1243,7 +1184,7 @@ func TestRunnerComposesAppendedMessagesIntoEveryRenderedPrompt(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
nil,
defaultArtifactReader(),
renderer,
client,
@@ -1310,7 +1251,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}}
runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil, nil)
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
PromptVersion: "1",
@@ -1385,14 +1326,13 @@ func TestRunnerRunPassesExtraParamsToGenerateRequestTarget(t *testing.T) {
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {
ID: "exec",
Endpoint: "http://profile/v1",
Model: "profile-model",
ExtraParams: extraParams,
ID: "exec", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://profile/v1",
Model: "profile-model",
ExtraParams: extraParams, TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds},
},
}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil)
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1418,7 +1358,7 @@ func TestRunnerRunAndPrepareResolveSameProfileAndEffectiveSettings(t *testing.T)
}}
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap"}}
runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil, nil)
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil, nil)
req := domain.RunRequest{
PromptID: "p",
@@ -1463,14 +1403,11 @@ func TestRunnerAdmissionUsesResolvedBackendIdentity(t *testing.T) {
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {
ID: "exec",
BackendID: "custom",
Model: "model",
ID: "exec", ResolvedTarget: &domain.ExecutionTarget{BackendID: "custom",
Model: "model", TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds},
},
}},
fakeBackendResolver{backends: map[string]domain.Backend{
"custom": {ID: "custom", Endpoint: "http://backend.example/v1"},
}},
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
@@ -1505,7 +1442,7 @@ func TestRunnerAdmissionUsesResolvedBackendIdentity(t *testing.T) {
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": defaultExecutionProfile(),
}},
nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
@@ -1565,11 +1502,9 @@ func TestRunnerAdmissionFailureSkipsCompletionCollaborators(t *testing.T) {
runner := NewRunnerWithRepairer(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", BackendID: "custom", Model: "model"},
}},
fakeBackendResolver{backends: map[string]domain.Backend{
"custom": {ID: "custom", Endpoint: "http://backend.example/v1"},
"exec": {ID: "exec", ResolvedTarget: &domain.ExecutionTarget{BackendID: "custom", Endpoint: "http://backend.example/v1", Model: "model"}},
}},
reader,
renderer,
llmClient,
@@ -1681,7 +1616,7 @@ func TestRunnerReleasesAdmissionAcrossRunOutcomes(t *testing.T) {
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": defaultExecutionProfile(),
}},
nil,
reader,
defaultRenderer(),
llmClient,
@@ -1715,8 +1650,8 @@ func TestRunnerRunExplicitProfileIDIsUsed(t *testing.T) {
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
promptRepo.def.DefaultProfile = "default-prof"
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"explicit-prof": {ID: "explicit-prof", Endpoint: "http://explicit/v1", Model: "explicit"},
"default-prof": {ID: "default-prof", Endpoint: "http://default/v1", Model: "default"},
"explicit-prof": {ID: "explicit-prof", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://explicit/v1", Model: "explicit", TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds}},
"default-prof": {ID: "default-prof", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://default/v1", Model: "default", TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds}},
}}
runner := newMinimalRunner(promptRepo, execRepo)
@@ -1740,7 +1675,7 @@ func TestRunnerRunPromptDefaultProfileIsUsedWhenNoExplicitProfileID(t *testing.T
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
promptRepo.def.DefaultProfile = "from-prompt"
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"from-prompt": {ID: "from-prompt", Endpoint: "http://llm/v1", Model: "m"},
"from-prompt": {ID: "from-prompt", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m", TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds}},
}}
runner := newMinimalRunner(promptRepo, execRepo)
@@ -1792,18 +1727,17 @@ func TestRunnerRunExplicitRuntimeOverrideBeatsSelectedProfileValue(t *testing.T)
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {
ID: "exec",
Endpoint: "http://profile/v1",
Model: "profile-model",
Temperature: 0.2,
MaxTokens: 500,
TopP: 0.9,
TimeoutSeconds: 120,
ServiceTier: "priority",
ID: "exec", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://profile/v1",
Model: "profile-model",
Temperature: 0.2,
MaxTokens: 500,
TopP: 0.9,
TimeoutSeconds: 120,
ServiceTier: "priority"},
},
}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil)
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1838,16 +1772,15 @@ func TestRunnerRunSelectedProfileBeatsBuiltInDefault(t *testing.T) {
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {
ID: "exec",
Endpoint: "http://profile/v1",
Model: "profile-model",
TopP: 0.8,
TimeoutSeconds: 90,
ServiceTier: "priority",
ID: "exec", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://profile/v1",
Model: "profile-model",
TopP: 0.8,
TimeoutSeconds: 90,
ServiceTier: "priority"},
},
}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil)
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1872,9 +1805,9 @@ func TestRunnerRunAPIKeyEnvResolvesFromEnvironment(t *testing.T) {
t.Setenv("PROMPTKIT_TEST_API_KEY", "secret")
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_TEST_API_KEY"},
"exec": {ID: "exec", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_TEST_API_KEY", TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds}},
}}
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil, nil)
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if err != nil {
@@ -1891,10 +1824,10 @@ func TestRunnerRunOptionalAPIKeyEnvMissingEnvironmentValueReachesLLM(t *testing.
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: environmentName},
"exec": {ID: "exec", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: environmentName, TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds}},
}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil)
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil)
result, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if err != nil || result == nil {
@@ -1912,10 +1845,10 @@ func TestRunnerRunDirectAPIKeyBypassesMissingEnvAndReachesLLM(t *testing.T) {
const directKey = "direct-runner-key"
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_MISSING_KEY"},
"exec": {ID: "exec", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_MISSING_KEY", TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds}},
}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil)
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil)
result, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1940,9 +1873,9 @@ func TestRunnerRunDirectAPIKeyBypassesMissingEnvAndReachesLLM(t *testing.T) {
func TestRunnerPrepareAPIKeyRequiredFailsWithoutDirectKey(t *testing.T) {
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true},
"exec": {ID: "exec", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true, TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds}},
}}
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil, nil)
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil, nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1958,10 +1891,10 @@ func TestRunnerRunAPIKeyRequiredSucceedsWithDirectKey(t *testing.T) {
const directKey = "direct-required-key"
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true},
"exec": {ID: "exec", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true, TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds}},
}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil)
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1986,9 +1919,9 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideWorks(t *testing.T) {
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"},
"exec": {ID: "exec", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://profile/v1", Model: "profile-model", TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds}},
}}
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil, nil)
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -2011,9 +1944,9 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideBeatsProfile(t *testing.T) {
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: profileEnv},
"exec": {ID: "exec", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: profileEnv, TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds}},
}}
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil, nil)
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -2035,9 +1968,9 @@ func TestRunnerRunAPIKeyValueNotPresentInMetadata(t *testing.T) {
t.Setenv(envName, secret)
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName},
"exec": {ID: "exec", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName, TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds}},
}}
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil, nil)
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if err != nil {
@@ -2053,7 +1986,7 @@ func TestRunnerRunAPIKeyValueNotPresentInMetadata(t *testing.T) {
}
func TestRunnerRunPromptLoadFailure(t *testing.T) {
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil, nil)
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"})
if !errors.Is(err, ErrPromptLoad) {
t.Fatalf("expected ErrPromptLoad, got %v", err)
@@ -2066,7 +1999,7 @@ func TestRunnerRunPromptLoadFailure(t *testing.T) {
func TestRunnerRunArtifactLoadFailure(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeArtifactReader{errByURI: map[string]error{"a://bad": errors.New("read failed")}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
@@ -2086,7 +2019,7 @@ func TestRunnerRunArtifactLoadFailure(t *testing.T) {
func TestRunnerRunPromptRenderFailure(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(),
&fakeRenderer{err: errors.New("render failed")},
@@ -2106,7 +2039,7 @@ func TestRunnerRunPromptRenderFailure(t *testing.T) {
func TestRunnerRunLLMFailure(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(),
defaultRenderer(),
@@ -2126,7 +2059,7 @@ func TestRunnerRunLLMFailure(t *testing.T) {
func TestRunnerRunCancellationPreservesGenerationCategory(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(),
defaultRenderer(),
@@ -2152,7 +2085,7 @@ func TestRunnerRunCancellationPreservesGenerationCategory(t *testing.T) {
func TestRunnerRunLLMInvalidRequestMapsToUsecaseInvalidRequest(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(),
defaultRenderer(),
@@ -2176,7 +2109,7 @@ func TestRunnerRunValidationStillWorks(t *testing.T) {
validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}}
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
defaultArtifactReader(),
defaultRenderer(),
@@ -2343,11 +2276,9 @@ func TestRunnerRepairStateMachine(t *testing.T) {
runner := NewRunnerWithRepairer(
&fakePromptRepo{def: definition},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", BackendID: "custom", Model: "profile-model"},
}},
fakeBackendResolver{backends: map[string]domain.Backend{
"custom": {ID: "custom", Endpoint: "http://backend.example/v1"},
"exec": {ID: "exec", ResolvedTarget: &domain.ExecutionTarget{BackendID: "custom", Endpoint: "http://backend.example/v1", Model: "profile-model"}},
}},
defaultArtifactReader(),
defaultRenderer(),
client,
@@ -2483,11 +2414,9 @@ func TestRunnerSchedulesInitialAndRepairGenerationThroughOneBackendPool(t *testi
runner := NewRunnerWithRepairer(
staticPromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
staticExecutionProfileRepo{profile: &domain.ExecutionProfile{
ID: "exec", BackendID: "custom", Model: "model",
}},
fakeBackendResolver{backends: map[string]domain.Backend{
"custom": {ID: "custom", Endpoint: "http://backend.example/v1"},
ID: "exec", ResolvedTarget: &domain.ExecutionTarget{BackendID: "custom", Endpoint: "http://backend.example/v1", Model: "model"},
}},
staticArtifactReader{},
staticRenderer{},
scheduledClient,
@@ -2556,220 +2485,6 @@ func TestRunnerSchedulesInitialAndRepairGenerationThroughOneBackendPool(t *testi
}
}
func TestExecutionProfileToTargetPopulatesAllFieldsAndCopiesExtraParams(t *testing.T) {
src := &domain.ExecutionProfile{
ID: "exec",
Endpoint: "http://profile/v1",
Model: "profile-model",
Temperature: 0.2,
MaxTokens: 123,
TopP: 0.75,
TimeoutSeconds: 90,
ServiceTier: "priority",
ReasoningEffort: "medium",
APIKeyEnv: "PROMPTKIT_API_KEY",
APIKeyRequired: true,
ExtraParams: map[string]any{
"provider_option": "on",
},
}
target := executionProfileToTarget(src)
if target.Endpoint != src.Endpoint ||
target.Model != src.Model ||
target.Temperature != src.Temperature ||
target.MaxTokens != src.MaxTokens ||
target.TopP != src.TopP ||
target.TimeoutSeconds != src.TimeoutSeconds ||
target.ServiceTier != src.ServiceTier ||
target.ReasoningEffort != src.ReasoningEffort ||
target.APIKeyEnv != src.APIKeyEnv ||
target.APIKeyRequired != src.APIKeyRequired {
t.Fatalf("expected all profile fields to populate target, got %+v", target)
}
if !reflect.DeepEqual(target.ExtraParams, src.ExtraParams) {
t.Fatalf("expected extra_params to match, got %#v", target.ExtraParams)
}
src.ExtraParams["provider_option"] = "changed"
if target.ExtraParams["provider_option"] != "on" {
t.Fatalf("expected extra_params copy to be independent, got %#v", target.ExtraParams)
}
}
func TestResolveExecutionTargetProfileValuesPopulateAllSupportedFields(t *testing.T) {
profileValue := &domain.ExecutionProfile{
ID: "exec",
Endpoint: "http://profile/v1",
Model: "profile-model",
Temperature: 0.3,
MaxTokens: 222,
TopP: 0.6,
TimeoutSeconds: 77,
ServiceTier: "priority",
ReasoningEffort: "low",
APIKeyEnv: "PROFILE_KEY",
ExtraParams: map[string]any{
"profile_option": "enabled",
},
}
target, presence := resolveExecutionTarget(nil, profileValue, nil)
if presence != (domain.ExecutionTargetPresence{}) {
t.Fatalf("expected no request override presence, got %+v", presence)
}
if target.Endpoint != profileValue.Endpoint ||
target.Model != profileValue.Model ||
target.Temperature != profileValue.Temperature ||
target.MaxTokens != profileValue.MaxTokens ||
target.TopP != profileValue.TopP ||
target.TimeoutSeconds != profileValue.TimeoutSeconds ||
target.ServiceTier != profileValue.ServiceTier ||
target.ReasoningEffort != profileValue.ReasoningEffort ||
target.APIKeyEnv != profileValue.APIKeyEnv ||
target.APIKeyRequired != profileValue.APIKeyRequired {
t.Fatalf("expected profile values to populate target, got %+v", target)
}
if !reflect.DeepEqual(target.ExtraParams, profileValue.ExtraParams) {
t.Fatalf("expected profile extra_params in target, got %#v", target.ExtraParams)
}
}
func TestResolveExecutionTargetRuntimeOverridesBeatProfileForAllOverrideableFields(t *testing.T) {
profileValue := &domain.ExecutionProfile{
ID: "exec",
Endpoint: "http://profile/v1",
Model: "profile-model",
Temperature: 0.2,
MaxTokens: 200,
TopP: 0.8,
TimeoutSeconds: 90,
ServiceTier: "priority",
ReasoningEffort: "medium",
APIKeyEnv: "PROFILE_KEY",
ExtraParams: map[string]any{
"profile_only": "yes",
},
}
override := &domain.ExecutionTargetOverride{
Endpoint: "http://override/v1",
Model: "override-model",
Temperature: float64Ptr(0.9),
MaxTokens: intPtr(111),
TopP: float64Ptr(0.5),
TimeoutSeconds: intPtr(30),
ServiceTier: "flex",
ReasoningEffort: stringPtr("high"),
APIKeyEnv: "RUNTIME_KEY",
ExtraParams: map[string]any{
"runtime_only": "yes",
},
}
target, presence := resolveExecutionTarget(nil, profileValue, override)
if presence != (domain.ExecutionTargetPresence{Temperature: true, MaxTokens: true, TopP: true, TimeoutSeconds: true}) {
t.Fatalf("unexpected override presence: %+v", presence)
}
if target.Endpoint != override.Endpoint ||
target.Model != override.Model ||
target.Temperature != *override.Temperature ||
target.MaxTokens != *override.MaxTokens ||
target.TopP != *override.TopP ||
target.TimeoutSeconds != *override.TimeoutSeconds ||
target.ServiceTier != override.ServiceTier ||
target.ReasoningEffort != *override.ReasoningEffort ||
target.APIKeyEnv != override.APIKeyEnv {
t.Fatalf("expected runtime overrides to win for all fields, got %+v", target)
}
if !reflect.DeepEqual(target.ExtraParams, override.ExtraParams) {
t.Fatalf("expected runtime extra_params to replace profile extra_params, got %#v", target.ExtraParams)
}
}
func TestResolveExecutionTargetReasoningOverrideStates(t *testing.T) {
profileValue := &domain.ExecutionProfile{
ReasoningEffort: "medium",
}
tests := []struct {
name string
override *string
want string
}{
{
name: "nil inherits profile value",
want: "medium",
},
{
name: "nonblank replaces and trims profile value",
override: stringPtr(" high "),
want: "high",
},
{
name: "blank clears profile value",
override: stringPtr(" \t "),
want: "",
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
target, _ := resolveExecutionTarget(nil, profileValue, &domain.ExecutionTargetOverride{
ReasoningEffort: tt.override,
})
if target.ReasoningEffort != tt.want {
t.Fatalf("reasoning effort = %q, want %q", target.ReasoningEffort, tt.want)
}
})
}
}
func TestMergeExecutionTargetEmptyStringOverridesDoNotErase(t *testing.T) {
base := domain.ExecutionTarget{
Endpoint: "http://base/v1",
Model: "base-model",
ServiceTier: "priority",
ReasoningEffort: "medium",
APIKeyEnv: "BASE_KEY",
}
override := domain.ExecutionTarget{
Endpoint: "http://override/v1",
Model: "override-model",
ServiceTier: " ",
ReasoningEffort: " ",
APIKeyEnv: "",
}
merged := mergeExecutionTarget(base, override)
if merged.Endpoint != "http://override/v1" || merged.Model != "override-model" {
t.Fatalf("expected endpoint/model to override, got %+v", merged)
}
if merged.ServiceTier != "priority" {
t.Fatalf("expected empty service_tier override to be ignored, got %q", merged.ServiceTier)
}
if merged.ReasoningEffort != "medium" {
t.Fatalf("expected empty reasoning_effort override to be ignored, got %q", merged.ReasoningEffort)
}
if merged.APIKeyEnv != "BASE_KEY" {
t.Fatalf("expected empty api_key_env override to be ignored, got %q", merged.APIKeyEnv)
}
}
func TestMergeExecutionTargetEmptyExtraParamsDoesNotErase(t *testing.T) {
base := domain.ExecutionTarget{
ExtraParams: map[string]any{
"keep": "value",
},
}
override := domain.ExecutionTarget{
ExtraParams: map[string]any{},
}
merged := mergeExecutionTarget(base, override)
if !reflect.DeepEqual(merged.ExtraParams, base.ExtraParams) {
t.Fatalf("expected empty extra_params override not to erase base values, got %#v", merged.ExtraParams)
}
}
func TestBuildOutputArtifactDefaults(t *testing.T) {
tests := []struct {
name string
@@ -2817,87 +2532,19 @@ func hashString(s string) string {
func defaultExecutionProfile() *domain.ExecutionProfile {
return &domain.ExecutionProfile{
ID: "exec",
Endpoint: "http://llm/v1",
Model: "model-from-profile",
ID: "exec", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://llm/v1",
Model: "model-from-profile", TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds},
}
}
func TestResolveExecutionTargetUsesBackendProfileAndRequestPrecedence(t *testing.T) {
backendValue := &domain.Backend{
ID: "custom",
Endpoint: "http://backend/v1",
APIKeyEnv: "BACKEND_KEY",
ExtraParams: map[string]any{"backend": true},
}
profileValue := &domain.ExecutionProfile{
ID: "exec",
BackendID: "custom",
Endpoint: "http://profile/v1",
Model: "profile-model",
APIKeyEnv: "PROFILE_KEY",
ExtraParams: map[string]any{"profile": true},
}
override := &domain.ExecutionTargetOverride{
Endpoint: "http://request/v1",
APIKeyEnv: "REQUEST_KEY",
ExtraParams: map[string]any{"request": true},
}
target, _ := resolveExecutionTarget(backendValue, profileValue, override)
if target.BackendID != "custom" {
t.Fatalf("endpoint override changed backend identity: %+v", target)
}
if target.Endpoint != "http://request/v1" || target.APIKeyEnv != "REQUEST_KEY" {
t.Fatalf("request values did not win: %+v", target)
}
if !reflect.DeepEqual(target.ExtraParams, map[string]any{"request": true}) {
t.Fatalf("expected whole-map request replacement, got %#v", target.ExtraParams)
}
target, _ = resolveExecutionTarget(backendValue, &domain.ExecutionProfile{
ID: "exec", BackendID: "custom", Model: "profile-model",
}, nil)
if target.Endpoint != backendValue.Endpoint ||
target.APIKeyEnv != backendValue.APIKeyEnv ||
!reflect.DeepEqual(target.ExtraParams, backendValue.ExtraParams) {
t.Fatalf("backend defaults were not inherited: %+v", target)
}
}
func TestRunnerPrepareBackendResolutionAndCredentialPrecedence(t *testing.T) {
resolver := fakeBackendResolver{backends: map[string]domain.Backend{
"custom": {ID: "custom", Endpoint: "http://backend/v1", APIKeyEnv: "BACKEND_KEY"},
}}
func TestRunnerPrepareCredentialPrecedence(t *testing.T) {
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
t.Run("unknown backend is a profile load failure", func(t *testing.T) {
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", BackendID: "unknown", Model: "model"},
}}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil, nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if !errors.Is(err, ErrProfileLoad) {
t.Fatalf("expected ErrProfileLoad, got %v", err)
}
})
t.Run("nil resolver is a profile load failure", func(t *testing.T) {
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", BackendID: "custom", Model: "model"},
}}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil, nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if !errors.Is(err, ErrProfileLoad) {
t.Fatalf("expected ErrProfileLoad, got %v", err)
}
})
t.Run("request environment wins", func(t *testing.T) {
t.Setenv("REQUEST_KEY", "request-secret")
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", BackendID: "custom", Model: "model", APIKeyEnv: "PROFILE_KEY"},
}}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil, nil)
"exec": {ID: "exec", ResolvedTarget: &domain.ExecutionTarget{BackendID: "custom", Endpoint: "http://backend/v1", Model: "model", APIKeyEnv: "PROFILE_KEY"}},
}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(),
@@ -2914,8 +2561,8 @@ func TestRunnerPrepareBackendResolutionAndCredentialPrecedence(t *testing.T) {
t.Run("required direct key clears inherited environment", func(t *testing.T) {
t.Setenv("BACKEND_KEY", "backend-secret")
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", BackendID: "custom", Model: "model", APIKeyRequired: true},
}}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil, nil)
"exec": {ID: "exec", ResolvedTarget: &domain.ExecutionTarget{BackendID: "custom", Endpoint: "http://backend/v1", Model: "model", APIKeyRequired: true}},
}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil, nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if !errors.Is(err, ErrAPIKeyRequired) {
@@ -2962,7 +2609,7 @@ func intPtr(v int) *int {
func newMinimalRunner(promptRepo *fakePromptRepo, execRepo *fakeExecutionProfileRepo) *Runner {
return NewRunner(
promptRepo,
execRepo, nil,
execRepo,
defaultArtifactReader(),
defaultRenderer(),

28
observation_boundary_contract_test.go
View File

@@ -7,8 +7,8 @@ import (
"testing"
"time"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit"
"gitea.maximumdirect.net/eric/promptkit/testfixture"
)
type observationClientFunc func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error)
@@ -24,16 +24,22 @@ func TestDelegatedOperationsIsolateObservations(t *testing.T) {
ctx, cancel := context.WithTimeout(context.WithValue(context.Background(), contextKey{}, "caller-value"), time.Minute)
defer cancel()
deadline, _ := ctx.Deadline()
child, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("child", "profile", "child message"), "."),
testfixture.WithProfiles(testfixture.Profile{ID: "profile", Endpoint: "https://example.invalid/v1", Model: "child-model"}),
promptkit.WithLLMClient(observationClientFunc(func(callCtx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
gotDeadline, ok := callCtx.Deadline()
if !ok || gotDeadline != deadline || callCtx.Value(contextKey{}) != "caller-value" || callCtx.Done() != ctx.Done() {
t.Error("delegation lost caller context behavior")
}
return &promptkit.GenerateResponse{Content: "child", Usage: promptkit.TokenUsage{PromptTokens: 7}}, nil
})))
child, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("child execution", map[string]any{
"profiles": map[string]any{"profile": map[string]any{"endpoint": "https://example.invalid/v1", "model": "child-model"}},
"prompts": map[string]any{"child": map[string]any{"1": map[string]any{
"default_profile": "profile",
"messages": []any{map[string]any{"role": "user", "content": "child message"}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
}}},
}),
}}}, promptkit.WithLLMClient(observationClientFunc(func(callCtx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
gotDeadline, ok := callCtx.Deadline()
if !ok || gotDeadline != deadline || callCtx.Value(contextKey{}) != "caller-value" || callCtx.Done() != ctx.Done() {
t.Error("delegation lost caller context behavior")
}
return &promptkit.GenerateResponse{Content: "child", Usage: promptkit.TokenUsage{PromptTokens: 7}}, nil
})))
if err != nil {
t.Fatal(err)
}

13
observation_contract_test.go
View File

@@ -6,22 +6,13 @@ import (
"errors"
"strings"
"testing"
"testing/fstest"
"gitea.maximumdirect.net/eric/promptkit"
"gitea.maximumdirect.net/eric/promptkit/testfixture"
)
func observationEngine(t *testing.T, client promptkit.LLMClient) *promptkit.Engine {
t.Helper()
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "hello"), "."),
testfixture.WithProfiles(testfixture.Profile{ID: "profile", Endpoint: "https://example.invalid/v1", Model: "model"}),
promptkit.WithLLMClient(client))
if err != nil {
t.Fatal(err)
}
return engine
return newSimpleEngine(t, "hello", promptkit.WithLLMClient(client))
}
func TestRunObservationCoversFacadeErrors(t *testing.T) {
@@ -129,7 +120,7 @@ func TestRunObservationClassifiesInvalidOutputAndGenerationFailure(t *testing.T)
}
func TestRunObservationPromptFailureAndPanicCleanup(t *testing.T) {
engine, err := testfixture.NewEngine(testfixture.Config{}, testfixture.WithPromptFS(fstest.MapFS{}, "."))
engine, err := promptkit.NewEngine(promptkit.Config{})
if err != nil {
t.Fatal(err)
}

14
output_contract_contract_test.go
View File

@@ -6,22 +6,10 @@ import (
"testing"
"gitea.maximumdirect.net/eric/promptkit"
"gitea.maximumdirect.net/eric/promptkit/testfixture"
)
func TestPreparationRejectsInvalidOutputContractWithPublicError(t *testing.T) {
engine, err := testfixture.NewEngine(
testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
testfixture.WithProfiles(testfixture.Profile{
ID: "profile",
Endpoint: "http://example.test/v1",
Model: "model",
}),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
engine := newSimpleEngine(t, "message")
for _, tc := range []struct {
name string

103
prepared_execution_contract_test.go
View File

@@ -12,8 +12,8 @@ import (
"testing/fstest"
"time"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit"
"gitea.maximumdirect.net/eric/promptkit/testfixture"
)
func TestPreparedExecutionFreezesSourcesAndReturnsIndependentDetails(t *testing.T) {
@@ -27,10 +27,17 @@ func TestPreparedExecutionFreezesSourcesAndReturnsIndependentDetails(t *testing.
client := &preparedRecordingClient{
response: &promptkit.GenerateResponse{Content: `{"value":3}`},
}
engine, err := testfixture.NewEngine(
testfixture.Config{},
testfixture.WithPromptFS(promptSource, "."),
testfixture.WithProfileFS(profileSource, "."),
promptInputs, err := promptkit.FilesystemInputs("prompts", promptSource, ".")
if err != nil {
t.Fatal(err)
}
profileInputs, err := promptkit.FilesystemInputs("profiles", profileSource, ".")
if err != nil {
t.Fatal(err)
}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{
{Inputs: profileInputs}, {Inputs: promptInputs},
}}},
promptkit.WithSchemaFS(schemaSource, "."),
promptkit.WithArtifactReader(reader),
promptkit.WithLLMClient(client),
@@ -431,17 +438,7 @@ func TestPreparedExecutionDiscardAndFormattingDoNotExposePrivateState(t *testing
client := &preparedRecordingClient{
response: &promptkit.GenerateResponse{Content: "generated output"},
}
engine, err := testfixture.NewEngine(
testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prepared", "profile", renderedContent), "."),
testfixture.WithProfiles(testfixture.Profile{
ID: "profile",
Endpoint: "http://example.test/v1",
Model: "model",
APIKeyRequired: true,
}),
promptkit.WithLLMClient(client),
)
engine, err := promptkit.NewEngine(preparedContractConfig(renderedContent, true), promptkit.WithLLMClient(client))
if err != nil {
t.Fatalf("construct engine: %v", err)
}
@@ -565,17 +562,7 @@ func TestPreparedExecutionCredentialCapacityAndTimingBoundaries(t *testing.T) {
client := &preparedRecordingClient{
response: &promptkit.GenerateResponse{Content: "unexpected"},
}
engine, err := testfixture.NewEngine(
testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prepared", "profile", "content"), "."),
testfixture.WithProfiles(testfixture.Profile{
ID: "profile",
Endpoint: "http://example.test/v1",
Model: "model",
APIKeyRequired: true,
}),
promptkit.WithLLMClient(client),
)
engine, err := promptkit.NewEngine(preparedContractConfig("content", true), promptkit.WithLLMClient(client))
if err != nil {
t.Fatalf("construct credential engine: %v", err)
}
@@ -767,47 +754,55 @@ func newPreparedContractEngine(
message string,
) *promptkit.Engine {
t.Helper()
engine, err := testfixture.NewEngine(
testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prepared", "profile", message), "."),
testfixture.WithProfiles(testfixture.Profile{
ID: "profile",
Endpoint: "http://example.test/v1",
Model: "model",
}),
promptkit.WithLLMClient(client),
)
engine, err := promptkit.NewEngine(preparedContractConfig(message, false), promptkit.WithLLMClient(client))
if err != nil {
t.Fatalf("construct prepared execution engine: %v", err)
}
return engine
}
func preparedContractConfig(message string, keyRequired bool) promptkit.Config {
return promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("prepared execution", map[string]any{
"profiles": map[string]any{"profile": map[string]any{
"endpoint": "http://example.test/v1", "model": "model", "api_key_required": keyRequired,
}},
"prompts": map[string]any{"prepared": map[string]any{"1": map[string]any{
"default_profile": "profile",
"messages": []any{map[string]any{"role": "user", "content": message}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
}}},
}),
}}}
}
func preparedPromptSource(label string) fstest.MapFS {
return fstest.MapFS{
"prompt.yaml": &fstest.MapFile{Data: []byte(`id: prepared
version: "1"
default_profile: profile
inputs:
- name: input
required: true
messages:
- role: user
content: 'Input={{input "input"}} Label={{.label}}'
description: ` + label + `
output:
format: json
validation_mode: json_schema
schema_path: schema.json
"prompt.yaml": &fstest.MapFile{Data: []byte(`prompts:
prepared:
"1":
default_profile: profile
inputs:
- name: input
required: true
messages:
- role: user
content: 'Input={{input "input"}} Label={{.label}}'
description: ` + label + `
output:
format: json
validation_mode: json_schema
schema_path: schema.json
`)},
}
}
func preparedProfileSource(model string) fstest.MapFS {
return fstest.MapFS{
"profile.yaml": &fstest.MapFile{Data: []byte(`id: profile
endpoint: http://example.test/v1
model: ` + model + `
"profile.yaml": &fstest.MapFile{Data: []byte(`profiles:
profile:
endpoint: http://example.test/v1
model: ` + model + `
`)},
}
}

18
prepared_observation_contract_test.go
View File

@@ -6,8 +6,8 @@ import (
"sync"
"testing"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit"
"gitea.maximumdirect.net/eric/promptkit/testfixture"
)
func TestPreparedObservationStartsOnlyAfterClaimAndKeepsObserver(t *testing.T) {
@@ -132,10 +132,18 @@ func TestPreparedObservationClaimRaceAndPanicCleanup(t *testing.T) {
func TestPreparedObservationFinishesAfterCredentialAndCapacityFailures(t *testing.T) {
const envName = "PROMPTKIT_PREPARED_OBSERVATION_KEY"
t.Setenv(envName, "available")
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prepared", "profile", "hello"), "."),
testfixture.WithProfiles(testfixture.Profile{ID: "profile", Endpoint: "https://example.invalid/v1", Model: "model", APIKeyRequired: true}),
promptkit.WithLLMClient(&fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}))
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("credential preparation", map[string]any{
"profiles": map[string]any{"profile": map[string]any{
"endpoint": "https://example.invalid/v1", "model": "model", "api_key_required": true,
}},
"prompts": map[string]any{"prepared": map[string]any{"1": map[string]any{
"default_profile": "profile",
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
}}},
}),
}}}, promptkit.WithLLMClient(&fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}))
if err != nil {
t.Fatal(err)
}

276
profile_inheritance_contract_test.go
View File

@@ -4,247 +4,169 @@ import (
"context"
"errors"
"io/fs"
"strings"
"sync"
"testing"
"testing/fstest"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit"
"gitea.maximumdirect.net/eric/promptkit/testfixture"
)
func aliasPrompt() map[string]any {
return map[string]any{"prompts": map[string]any{"prepared": map[string]any{"1": map[string]any{
"default_profile": "child",
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
}}}}
}
func TestProfileInheritanceBuiltInAliasWorkflow(t *testing.T) {
engine, err := testfixture.NewEngine(testfixture.Config{
PromptDir: frameworkPromptDir,
SchemaDir: frameworkSchemaDir,
}, testfixture.WithProfiles(testfixture.Profile{
ID: "weather-light",
BaseProfileID: "promptkit-openrouter-standard",
ReasoningEffort: "high",
TimeoutSeconds: 120,
}))
standardID, reasoning, timeout := "promptkit-openrouter-standard", "high", 120
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("application alias", map[string]promptkit.ProfileConfig{
"child": {Extends: &standardID, ReasoningEffort: &reasoning, TimeoutSeconds: &timeout},
})}}}},
Main: &configkit.Input{Source: configkit.Mapping("prompt", aliasPrompt())},
}})
if err != nil {
t.Fatalf("construct alias engine: %v", err)
}
base, err := engine.InspectProfile(context.Background(), "promptkit-openrouter-standard")
base, err := engine.InspectProfile(context.Background(), standardID)
if err != nil {
t.Fatalf("inspect base: %v", err)
}
child, err := engine.InspectProfile(context.Background(), "weather-light")
child, err := engine.InspectProfile(context.Background(), "child")
if err != nil {
t.Fatalf("inspect alias: %v", err)
}
if child.ProfileID != "weather-light" ||
child.EffectiveModelParams.BackendID != base.EffectiveModelParams.BackendID ||
if child.ProfileID != "child" || child.EffectiveModelParams.BackendID != base.EffectiveModelParams.BackendID ||
child.EffectiveModelParams.Model != base.EffectiveModelParams.Model ||
child.EffectiveModelParams.ReasoningEffort != "high" ||
child.EffectiveModelParams.TimeoutSeconds != 120 {
child.EffectiveModelParams.ReasoningEffort != reasoning || child.EffectiveModelParams.TimeoutSeconds != timeout {
t.Fatalf("alias inspection = %+v, base = %+v", child, base)
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: frameworkMarkdownSummaryPromptID,
ProfileID: "weather-light",
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline("Rin opens the gate."),
"glossary": promptkit.Inline("gate: A guarded passage."),
},
})
if err != nil {
t.Fatalf("prepare alias: %v", err)
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
if err != nil || prepared.SelectedProfileID != "child" {
t.Fatalf("prepare alias = (%+v, %v)", prepared, err)
}
if prepared.SelectedProfileID != "weather-light" {
t.Fatalf("SelectedProfileID = %q", prepared.SelectedProfileID)
}
timeout := 15
reasoning := "low"
overridden, err := engine.Prepare(context.Background(), promptkit.RunRequest{
PromptID: frameworkMarkdownSummaryPromptID,
ProfileID: "weather-light",
Execution: &promptkit.ExecutionTargetOverride{
TimeoutSeconds: &timeout,
ReasoningEffort: &reasoning,
},
Inputs: map[string]promptkit.ArtifactRef{
"transcript": promptkit.Inline("Rin opens the gate."),
"glossary": promptkit.Inline("gate: A guarded passage."),
},
})
if err != nil {
t.Fatalf("prepare override: %v", err)
}
if overridden.EffectiveModelParams.TimeoutSeconds != timeout ||
overridden.EffectiveModelParams.ReasoningEffort != reasoning {
t.Fatalf("runtime override target = %+v", overridden.EffectiveModelParams)
shorter, lower := 15, "low"
overridden, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prepared", Execution: &promptkit.ExecutionTargetOverride{
TimeoutSeconds: &shorter, ReasoningEffort: &lower,
}})
if err != nil || overridden.EffectiveModelParams.TimeoutSeconds != shorter || overridden.EffectiveModelParams.ReasoningEffort != lower {
t.Fatalf("runtime override = (%+v, %v)", overridden, err)
}
}
func TestProfileInheritanceYAMLAliasOfBuiltIn(t *testing.T) {
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(fstest.MapFS{}, "."),
testfixture.WithProfileFS(fstest.MapFS{
"alias.yaml": &fstest.MapFile{Data: []byte("id: yaml-alias\nbase_profile: promptkit-openrouter-standard\n")},
}, "."),
)
inputs, err := promptkit.FilesystemInputs("profile alias", fstest.MapFS{
"config/alias.yaml": &fstest.MapFile{Data: []byte("profiles:\n yaml-alias:\n extends: promptkit-openrouter-standard\n")},
}, "config")
if err != nil {
t.Fatalf("construct YAML alias engine: %v", err)
t.Fatal(err)
}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}}})
if err != nil {
t.Fatal(err)
}
inspection, err := engine.InspectProfile(context.Background(), "yaml-alias")
if err != nil {
t.Fatalf("inspect YAML alias: %v", err)
}
if inspection.ProfileID != "yaml-alias" || inspection.EffectiveModelParams.Model == "" {
t.Fatalf("YAML alias inspection = %+v", inspection)
if err != nil || inspection.ProfileID != "yaml-alias" || inspection.EffectiveModelParams.Model == "" {
t.Fatalf("YAML alias inspection = (%+v, %v)", inspection, err)
}
}
func TestProfileInheritanceRetainsRequiredCredentialBehavior(t *testing.T) {
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(fstest.MapFS{}, "."),
testfixture.WithBackend(testfixture.Backend{
ID: "credential-backend",
Endpoint: "https://credential.example/v1",
APIKeyEnv: "OPTIONAL_BACKEND_KEY",
}),
testfixture.WithProfiles(
testfixture.Profile{
ID: "credential-base",
BackendID: "credential-backend",
Model: "model",
APIKeyRequired: true,
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("credentials", map[string]any{
"backends": map[string]any{"credential-backend": map[string]any{
"endpoint": "https://credential.example/v1", "api_key_env": "OPTIONAL_BACKEND_KEY",
}},
"profiles": map[string]any{
"credential-base": map[string]any{"backend": "credential-backend", "model": "model", "api_key_required": true},
"credential-child": map[string]any{"extends": "credential-base"},
},
testfixture.Profile{ID: "credential-child", BaseProfileID: "credential-base"},
),
)
}),
}}})
if err != nil {
t.Fatalf("construct credential inheritance engine: %v", err)
t.Fatal(err)
}
inspection, err := engine.InspectProfile(context.Background(), "credential-child")
if err != nil {
t.Fatalf("inspect credential child: %v", err)
}
if !inspection.APIKeyRequired || inspection.EffectiveModelParams.APIKeyEnv != "" {
t.Fatalf("credential inspection = %+v", inspection)
if err != nil || !inspection.APIKeyRequired || inspection.EffectiveModelParams.APIKeyEnv != "" {
t.Fatalf("credential inspection = (%+v, %v)", inspection, err)
}
}
func TestProfileInheritancePreservesPublicErrorIdentities(t *testing.T) {
newEngine := func(t *testing.T, profiles fs.FS) (*promptkit.Engine, error) {
t.Helper()
options := []any{testfixture.WithPromptFS(fstest.MapFS{}, ".")}
if profiles != nil {
options = append(options, testfixture.WithProfileFS(profiles, "."))
engine, err := promptkit.NewEngine(promptkit.Config{})
if err != nil {
t.Fatal(err)
}
if inspection, err := engine.InspectProfile(context.Background(), "missing"); inspection != nil || !errors.Is(err, promptkit.ErrProfileNotFound) {
t.Fatalf("missing profile = (%+v, %v)", inspection, err)
}
for _, document := range []map[string]any{
{"profiles": map[string]any{"child": map[string]any{"extends": "missing"}}},
{"profiles": map[string]any{"a": map[string]any{"extends": "b"}, "b": map[string]any{"extends": "a"}}},
} {
invalid, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("invalid inheritance", document),
}}})
if invalid != nil || !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("invalid inheritance = (%+v, %v)", invalid, err)
}
return testfixture.NewEngine(testfixture.Config{}, options...)
}
tests := []struct {
name string
profiles fs.FS
profile string
contains []string
want error
wantNot error
}{
{
name: "missing selected profile",
profile: "missing",
want: promptkit.ErrProfileNotFound,
wantNot: promptkit.ErrProfileLoad,
},
{
name: "missing base",
profiles: fstest.MapFS{
"child.yaml": &fstest.MapFile{Data: []byte("id: child\nbase_profile: missing\n")},
},
profile: "child",
contains: []string{"child", "missing"},
want: promptkit.ErrProfileLoad,
wantNot: promptkit.ErrProfileNotFound,
},
{
name: "cycle",
profiles: fstest.MapFS{
"a.yaml": &fstest.MapFile{Data: []byte("id: a\nbase_profile: b\n")},
"b.yaml": &fstest.MapFile{Data: []byte("id: b\nbase_profile: a\n")},
},
profile: "a",
contains: []string{"a", "b"},
want: promptkit.ErrProfileLoad,
},
}
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
engine, err := newEngine(t, tc.profiles)
if tc.profiles != nil {
if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("construction = (%#v, %v), want invalid configuration", engine, err)
}
return
}
if err != nil {
t.Fatalf("construct engine: %v", err)
}
result, err := engine.InspectProfile(context.Background(), tc.profile)
if result != nil || !errors.Is(err, tc.want) || (tc.wantNot != nil && errors.Is(err, tc.wantNot)) {
t.Fatalf("inspection=(%+v, %v), want %v without %v", result, err, tc.want, tc.wantNot)
}
for _, fragment := range tc.contains {
if !strings.Contains(err.Error(), fragment) {
t.Fatalf("error = %v, want %q", err, fragment)
}
}
})
}
}
func TestProfileInheritanceFreezesPreparedExecution(t *testing.T) {
profiles := &mutableInheritanceProfileFS{files: fstest.MapFS{
"child.yaml": &fstest.MapFile{Data: []byte("id: child\nbase_profile: base\n")},
"base.yaml": &fstest.MapFile{Data: []byte("id: base\nendpoint: https://base.example/v1\nmodel: first-model\n")},
"config/profiles.yaml": &fstest.MapFile{Data: []byte("profiles:\n base:\n endpoint: https://base.example/v1\n model: first-model\n child:\n extends: base\n")},
}}
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
engine, err := testfixture.NewEngine(testfixture.Config{},
testfixture.WithPromptFS(contractPromptFS("prepared", "child", "content"), "."),
testfixture.WithProfileFS(profiles, "."),
promptkit.WithLLMClient(client),
)
inputs, err := promptkit.FilesystemInputs("profiles", profiles, "config")
if err != nil {
t.Fatalf("construct engine: %v", err)
t.Fatal(err)
}
client := &recordingProfileClient{}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: inputs}},
Main: &configkit.Input{Source: configkit.Mapping("prompt", aliasPrompt())},
}}, promptkit.WithLLMClient(client))
if err != nil {
t.Fatal(err)
}
prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"})
if err != nil {
t.Fatalf("prepare execution: %v", err)
t.Fatal(err)
}
profiles.set("base.yaml", "id: base\nendpoint: https://base.example/v1\nmodel: second-model\n")
profiles.set("config/profiles.yaml", "profiles:\n base:\n endpoint: https://base.example/v1\n model: second-model\n child:\n extends: base\n")
result, err := engine.RunPrepared(context.Background(), prepared)
if err != nil || result == nil || len(client.requests) != 1 || client.requests[0].Target.Model != "first-model" {
t.Fatalf("prepared execution=(%+v, %v), requests=%+v", result, err, client.requests)
if err != nil || result == nil || client.model != "first-model" {
t.Fatalf("prepared execution = (%+v, %v), model %q", result, err, client.model)
}
inspection, err := engine.InspectProfile(context.Background(), "child")
if err != nil || inspection.EffectiveModelParams.Model != "first-model" {
t.Fatalf("existing engine inspection=(%+v, %v)", inspection, err)
t.Fatalf("existing engine inspection = (%+v, %v)", inspection, err)
}
}
type recordingProfileClient struct{ model string }
func (client *recordingProfileClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) {
client.model = request.Target.Model
return &promptkit.GenerateResponse{Content: "ok"}, ctx.Err()
}
type mutableInheritanceProfileFS struct {
mu sync.RWMutex
files fstest.MapFS
}
func (f *mutableInheritanceProfileFS) Open(name string) (fs.File, error) {
f.mu.RLock()
defer f.mu.RUnlock()
return f.files.Open(name)
func (source *mutableInheritanceProfileFS) Open(name string) (fs.File, error) {
source.mu.RLock()
defer source.mu.RUnlock()
return source.files.Open(name)
}
func (f *mutableInheritanceProfileFS) set(name, content string) {
f.mu.Lock()
defer f.mu.Unlock()
f.files[name] = &fstest.MapFile{Data: []byte(content)}
func (source *mutableInheritanceProfileFS) set(name, content string) {
source.mu.Lock()
defer source.mu.Unlock()
source.files[name] = &fstest.MapFile{Data: []byte(content)}
}

99
profile_inspection_contract_test.go Normal file
View File

@@ -0,0 +1,99 @@
package promptkit_test
import (
"context"
"errors"
"testing"
"testing/fstest"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit"
)
func TestInspectProfileUsesLiteralFileKeyForPreparation(t *testing.T) {
profiles, err := promptkit.FilesystemInputs("literal profile", fstest.MapFS{
"config/profile.yaml": &fstest.MapFile{Data: []byte("profiles:\n ' literal ':\n endpoint: https://example.test/v1\n model: model\n")},
}, "config")
if err != nil {
t.Fatal(err)
}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{
Underlays: []configkit.Underlay{{Inputs: profiles}},
Main: &configkit.Input{Source: configkit.Mapping("prompt", map[string]any{
"prompts": map[string]any{"prompt": map[string]any{"1": map[string]any{
"default_profile": " literal ",
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
}}},
})},
}})
if err != nil {
t.Fatal(err)
}
inspected, err := engine.InspectProfile(context.Background(), " literal ")
if err != nil {
t.Fatal(err)
}
prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if err != nil {
t.Fatal(err)
}
if inspected.ProfileID != " literal " || prepared.SelectedProfileID != inspected.ProfileID || prepared.EffectiveModelParams.Model != inspected.EffectiveModelParams.Model {
t.Fatalf("inspection=%+v preparation=%+v", inspected, prepared)
}
if _, err := engine.InspectProfile(context.Background(), "literal"); !errors.Is(err, promptkit.ErrProfileNotFound) {
t.Fatalf("trimmed ID resolved unexpectedly: %v", err)
}
}
func TestInspectProfileIsIndependentAndDoesNotInvokeProvider(t *testing.T) {
client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("inspection", map[string]any{"profiles": map[string]any{
"profile": map[string]any{
"endpoint": "https://example.test/v1", "model": "model", "api_key_required": true,
"extra_params": map[string]any{"nested": map[string]any{"value": "original"}},
},
}}),
}}}, promptkit.WithLLMClient(client))
if err != nil {
t.Fatal(err)
}
first, err := engine.InspectProfile(context.Background(), "profile")
if err != nil || first == nil || !first.APIKeyRequired {
t.Fatalf("first inspection=(%+v, %v)", first, err)
}
first.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["value"] = "changed"
second, err := engine.InspectProfile(context.Background(), "profile")
if err != nil || second == nil || second.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["value"] != "original" || len(client.requests) != 0 {
t.Fatalf("inspection changed snapshot or invoked provider: (%+v, %v), calls=%d", second, err, len(client.requests))
}
}
func TestInspectProfilePublicErrorsAndPartialTarget(t *testing.T) {
var nilEngine *promptkit.Engine
if _, err := nilEngine.InspectProfile(context.Background(), "profile"); !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("nil engine: %v", err)
}
engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{
Source: configkit.Mapping("partial", map[string]any{"profiles": map[string]any{"profile": map[string]any{"model": "model"}}}),
}}})
if err != nil {
t.Fatal(err)
}
if _, err := engine.InspectProfile(context.Background(), " "); !errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("blank selection: %v", err)
}
if _, err := engine.InspectProfile(context.Background(), "missing"); !errors.Is(err, promptkit.ErrProfileNotFound) {
t.Fatalf("missing selection: %v", err)
}
ctx, cancel := context.WithCancel(context.Background())
cancel()
if _, err := engine.InspectProfile(ctx, "profile"); !errors.Is(err, context.Canceled) {
t.Fatalf("canceled inspection: %v", err)
}
partial, err := engine.InspectProfile(context.Background(), "profile")
if err != nil || partial == nil || partial.EffectiveModelParams.Endpoint != "" || partial.EffectiveModelParams.Model != "model" {
t.Fatalf("partial inspection=(%+v, %v)", partial, err)
}
}

1497
public_contract_test.go
View File

File diff suppressed because it is too large Load Diff

99
reliable_execution_contract_test.go
View File

@@ -12,8 +12,8 @@ import (
"testing/fstest"
"time"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit"
"gitea.maximumdirect.net/eric/promptkit/testfixture"
)
func observedHTTPResponse(status int, body string) *http.Response {
@@ -29,7 +29,7 @@ func TestPublicRetryRepairSequenceKeepsUsageAndEventsConsistent(t *testing.T) {
}
var calls int
one := 1
config := contractConfig(frameworkSchemaDir)
config := simpleConfig("message")
config.TransientRetries = &one
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
if calls >= len(responses) {
@@ -40,12 +40,12 @@ func TestPublicRetryRepairSequenceKeepsUsageAndEventsConsistent(t *testing.T) {
calls++
return r, nil
})}
engine, err := testfixture.NewEngine(config)
engine, err := promptkit.NewEngine(config)
if err != nil {
t.Fatal(err)
}
var events []promptkit.ExecutionEvent
req := generationErrorRunRequest()
req := simpleGenerationRequest()
req.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: 1}
req.Observer = func(event promptkit.ExecutionEvent) { events = append(events, event) }
result, err := engine.Run(context.Background(), req)
@@ -82,6 +82,18 @@ func TestPublicRetryRepairSequenceKeepsUsageAndEventsConsistent(t *testing.T) {
}
}
func reliabilityConfig(backends, profiles map[string]any) promptkit.Config {
return promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Source: configkit.Mapping("reliable execution", map[string]any{
"backends": backends,
"profiles": profiles,
"prompts": map[string]any{"prompt": map[string]any{"1": map[string]any{
"default_profile": "profile-a",
"messages": []any{map[string]any{"role": "user", "content": "hello"}},
"output": map[string]any{"format": "text", "validation_mode": "none"},
}}},
})}}}
}
func TestProfilesSharingBackendCoordinateRetryRepairAndAdmission(t *testing.T) {
firstA := make(chan struct{})
enteredB := make(chan struct{})
@@ -125,10 +137,14 @@ func TestProfilesSharingBackendCoordinateRetryRepairAndAdmission(t *testing.T) {
}
})
one := 1
engine, err := testfixture.NewEngine(testfixture.Config{HTTPClient: &http.Client{Transport: transport}, TransientRetries: &one},
testfixture.WithPromptFS(contractPromptFS("prompt", "profile-a", "hello"), "."),
testfixture.WithBackend(testfixture.Backend{ID: "limited", Endpoint: "https://example.invalid/v1", ConcurrencyLimit: 1, QueueCapacity: &one}),
testfixture.WithProfiles(testfixture.Profile{ID: "profile-a", BackendID: "limited", Model: "model-a"}, testfixture.Profile{ID: "profile-b", BackendID: "limited", Model: "model-b"}))
config := reliabilityConfig(map[string]any{"limited": map[string]any{
"endpoint": "https://example.invalid/v1", "concurrency_limit": 1, "queue_capacity": 1,
}}, map[string]any{
"profile-a": map[string]any{"backend": "limited", "model": "model-a"},
"profile-b": map[string]any{"backend": "limited", "model": "model-b"},
})
config.HTTPClient, config.TransientRetries = &http.Client{Transport: transport}, &one
engine, err := promptkit.NewEngine(config)
if err != nil {
t.Fatal(err)
}
@@ -187,11 +203,15 @@ func TestDifferentBackendPoolsCanDispatchTogether(t *testing.T) {
return observedHTTPResponse(200, `{"choices":[{"message":{"content":"ok"}}]}`), nil
})
zero := 0
engine, err := testfixture.NewEngine(testfixture.Config{HTTPClient: &http.Client{Transport: transport}, TransientRetries: &zero},
testfixture.WithPromptFS(contractPromptFS("prompt", "profile-a", "hello"), "."),
testfixture.WithBackend(testfixture.Backend{ID: "backend-a", Endpoint: "https://example.invalid/v1", ConcurrencyLimit: 1, QueueCapacity: &zero}),
testfixture.WithBackend(testfixture.Backend{ID: "backend-b", Endpoint: "https://example.invalid/v1", ConcurrencyLimit: 1, QueueCapacity: &zero}),
testfixture.WithProfiles(testfixture.Profile{ID: "profile-a", BackendID: "backend-a", Model: "model-a"}, testfixture.Profile{ID: "profile-b", BackendID: "backend-b", Model: "model-b"}))
config := reliabilityConfig(map[string]any{
"backend-a": map[string]any{"endpoint": "https://example.invalid/v1", "concurrency_limit": 1, "queue_capacity": 0},
"backend-b": map[string]any{"endpoint": "https://example.invalid/v1", "concurrency_limit": 1, "queue_capacity": 0},
}, map[string]any{
"profile-a": map[string]any{"backend": "backend-a", "model": "model-a"},
"profile-b": map[string]any{"backend": "backend-b", "model": "model-b"},
})
config.HTTPClient, config.TransientRetries = &http.Client{Transport: transport}, &zero
engine, err := promptkit.NewEngine(config)
if err != nil {
t.Fatal(err)
}
@@ -230,7 +250,17 @@ func TestDifferentBackendPoolsCanDispatchTogether(t *testing.T) {
func TestPreparedRetriesFreezeSourceAndCredentialAtExecution(t *testing.T) {
const envName = "PROMPTKIT_PREPARED_RETRY_KEY"
t.Setenv(envName, "first-key")
source := contractPromptFS("prompt", "profile", "original message")
source := fstest.MapFS{"prompt.yaml": &fstest.MapFile{Data: []byte(`prompts:
prompt:
"1":
default_profile: profile
messages:
- role: user
content: original message
output:
format: text
validation_mode: none
`)}}
var calls int
var auth, content []string
transport := roundTripFunc(func(req *http.Request) (*http.Response, error) {
@@ -252,8 +282,20 @@ func TestPreparedRetriesFreezeSourceAndCredentialAtExecution(t *testing.T) {
return observedHTTPResponse(200, `{"choices":[{"message":{"content":"ok"}}]}`), nil
})
one := 1
engine, err := testfixture.NewEngine(testfixture.Config{HTTPClient: &http.Client{Transport: transport}, TransientRetries: &one},
testfixture.WithPromptFS(source, "."), testfixture.WithProfiles(testfixture.Profile{ID: "profile", Endpoint: "https://example.invalid/v1", Model: "model"}))
inputs, err := promptkit.FilesystemInputs("prompts", source, ".")
if err != nil {
t.Fatal(err)
}
endpoint, model := "https://example.invalid/v1", "model"
engine, err := promptkit.NewEngine(promptkit.Config{
Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{
{Inputs: inputs},
{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile", map[string]promptkit.ProfileConfig{
"profile": {Endpoint: &endpoint, Model: &model},
})}}},
}},
HTTPClient: &http.Client{Transport: transport}, TransientRetries: &one,
})
if err != nil {
t.Fatal(err)
}
@@ -262,15 +304,16 @@ func TestPreparedRetriesFreezeSourceAndCredentialAtExecution(t *testing.T) {
if err != nil {
t.Fatal(err)
}
source["prompt.yaml"] = &fstest.MapFile{Data: []byte(`id: prompt
version: "1"
default_profile: profile
messages:
- role: user
content: changed message
output:
format: text
validation_mode: none
source["prompt.yaml"] = &fstest.MapFile{Data: []byte(`prompts:
prompt:
"1":
default_profile: profile
messages:
- role: user
content: changed message
output:
format: text
validation_mode: none
`)}
result, err := engine.RunPrepared(context.Background(), prepared)
if err != nil || result == nil || calls != 2 {
@@ -290,7 +333,7 @@ func TestPublicNonretryableAndInjectedAttemptsStaySingle(t *testing.T) {
three := 3
engine, calls := retryTestEngine(t, &three, func(int) *http.Response { return retryHTTPResponse(400, "") })
var events []promptkit.ExecutionEvent
req := generationErrorRunRequest()
req := simpleGenerationRequest()
req.Observer = func(event promptkit.ExecutionEvent) { events = append(events, event) }
result, err := engine.Run(context.Background(), req)
if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) || errors.Is(err, promptkit.ErrRetriesExhausted) || *calls != 1 || len(events) != 3 {
@@ -314,7 +357,7 @@ func TestPublicExhaustionObservationRetainsProviderCause(t *testing.T) {
one := 1
engine, calls := retryTestEngine(t, &one, func(int) *http.Response { return retryHTTPResponse(503, "") })
var events []promptkit.ExecutionEvent
req := generationErrorRunRequest()
req := simpleGenerationRequest()
req.Observer = func(event promptkit.ExecutionEvent) { events = append(events, event) }
result, err := engine.Run(context.Background(), req)
var exhausted *promptkit.RetryExhaustedError
@@ -334,7 +377,7 @@ func TestPublicObserverCancellationStopsBeforeRetry(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
var events []promptkit.ExecutionEvent
req := generationErrorRunRequest()
req := simpleGenerationRequest()
req.Observer = func(event promptkit.ExecutionEvent) {
events = append(events, event)
if event.Kind == promptkit.ExecutionAttempt {

2
testdata/framework/fixtures/glossary.yml vendored
View File

@@ -1,2 +0,0 @@
archive: A catalogued collection of written records.
marker: A small label used to classify an entry.

2
testdata/framework/fixtures/transcript.md vendored
View File

@@ -1,2 +0,0 @@
Nia labels the archive.
The archive receives a blue marker.

7
testdata/framework/profiles/contract-fast.yaml vendored
View File

@@ -1,7 +0,0 @@
id: contract-fast
endpoint: http://localhost:8000/v1
model: contract-fast-model
temperature: 0.2
max_tokens: 500
top_p: 1
timeout_seconds: 90

7
testdata/framework/profiles/contract-quality.yaml vendored
View File

@@ -1,7 +0,0 @@
id: contract-quality
endpoint: http://localhost:8000/v1
model: contract-quality-model
temperature: 0.1
max_tokens: 1000
top_p: 0.9
timeout_seconds: 120

1
testdata/framework/prompts/contract.markdown_summary.system.md vendored
View File

@@ -1 +0,0 @@
You summarize synthetic archive notes in clear Markdown.

7
testdata/framework/prompts/contract.markdown_summary.user.md vendored
View File

@@ -1,7 +0,0 @@
Summarize this transcript:
{{input "transcript"}}
Optional glossary:
{{input "glossary"}}

20
testdata/framework/prompts/contract.markdown_summary.yaml vendored
View File

@@ -1,20 +0,0 @@
id: contract.markdown_summary
version: "1.0.0"
default_profile: contract-fast
description: Summarize a synthetic transcript in Markdown.
inputs:
- name: transcript
required: true
content_type: text/markdown
- name: glossary
required: false
content_type: text/yaml
messages:
- role: system
content_file: ./contract.markdown_summary.system.md
- role: user
content_file: ./contract.markdown_summary.user.md
output:
format: markdown
validation_mode: basic
repair_attempts: 0

1
testdata/framework/prompts/contract.structured_events.system.md vendored
View File

@@ -1 +0,0 @@
Return only JSON that satisfies the requested event schema.

7
testdata/framework/prompts/contract.structured_events.user.md vendored
View File

@@ -1,7 +0,0 @@
Extract events from this transcript:
{{input "transcript"}}
Optional glossary:
{{input "glossary"}}

21
testdata/framework/prompts/contract.structured_events.yaml vendored
View File

@@ -1,21 +0,0 @@
id: contract.structured_events
version: "1.0.0"
default_profile: contract-quality
description: Extract synthetic events as structured JSON.
inputs:
- name: transcript
required: true
content_type: text/markdown
- name: glossary
required: false
content_type: text/yaml
messages:
- role: system
content_file: ./contract.structured_events.system.md
- role: user
content_file: ./contract.structured_events.user.md
output:
format: json
validation_mode: json_schema
schema_path: structured_events.schema.json
repair_attempts: 0

19
testdata/framework/schemas/structured_events.schema.json vendored
View File

@@ -1,19 +0,0 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"type": "object",
"required": ["events"],
"properties": {
"events": {
"type": "array",
"items": {
"type": "object",
"required": ["title"],
"properties": {
"title": {"type": "string"}
},
"additionalProperties": false
}
}
},
"additionalProperties": false
}

359
testfixture/engine.go
View File

@@ -1,359 +0,0 @@
// Package testfixture keeps existing runtime regression scenarios executable
// while their source setup is expressed through the new configuration boundary.
package testfixture
import (
"bytes"
"fmt"
"io"
"io/fs"
"net/http"
"os"
"path/filepath"
"strings"
"time"
"gitea.maximumdirect.net/eric/ConfigKit"
"gitea.maximumdirect.net/eric/promptkit"
"gopkg.in/yaml.v3"
)
// Config carries the existing regression fixtures' construction settings.
type Config struct {
PromptDir string
ProfileDir string
SchemaDir string
Timeout time.Duration
HTTPClient *http.Client
TransientRetries *int
}
// Profile describes an existing regression fixture's execution settings.
type Profile struct {
ID, BaseProfileID, BackendID, Endpoint, Model string
Temperature, TopP float64
MaxTokens, TimeoutSeconds int
ServiceTier, ReasoningEffort string
APIKeyRequired bool
ExtraParams map[string]any
}
// OpenAICompatibleProfileConfig describes a profile fixture.
type OpenAICompatibleProfileConfig = Profile
// OpenAICompatibleProfile produces a profile fixture.
func OpenAICompatibleProfile(config OpenAICompatibleProfileConfig) Profile {
profile := Profile(config)
if config.ExtraParams != nil {
profile.ExtraParams = make(map[string]any, len(config.ExtraParams))
for key, value := range config.ExtraParams {
profile.ExtraParams[key] = value
}
}
return profile
}
// Backend describes an existing regression fixture's backend settings.
type Backend struct {
ID, Endpoint, APIKeyEnv string
ConcurrencyLimit int
QueueCapacity *int
ExtraParams map[string]any
}
// LocalBackend returns a conventional local backend fixture.
func LocalBackend(endpoint string, concurrencyLimit int) Backend {
return Backend{ID: promptkit.BackendLocal, Endpoint: endpoint, ConcurrencyLimit: concurrencyLimit}
}
type option func(*choices) error
type choices struct {
promptFS, profileFS, fallbackFS fs.FS
promptRoot, profileRoot, fallbackRoot string
promptFile, profileFile string
profiles []Profile
profilesSet bool
backends []Backend
}
// WithPromptFS selects prompt fixture files.
func WithPromptFS(fsys fs.FS, root string) option {
return func(c *choices) error {
if err := validSourceFS(fsys, root); err != nil {
return err
}
c.promptFS, c.promptRoot, c.promptFile = fsys, root, ""
return nil
}
}
// WithPromptFile selects one prompt fixture file.
func WithPromptFile(name string) option {
return func(c *choices) error {
if strings.TrimSpace(name) == "" {
return fmt.Errorf("prompt file is required")
}
c.promptFS, c.promptFile = nil, name
return nil
}
}
// WithProfileFS selects profile fixture files.
func WithProfileFS(fsys fs.FS, root string) option {
return func(c *choices) error {
if err := validSourceFS(fsys, root); err != nil {
return err
}
c.profileFS, c.profileRoot, c.profileFile = fsys, root, ""
return nil
}
}
// WithProfileFile selects one profile fixture file.
func WithProfileFile(name string) option {
return func(c *choices) error {
if strings.TrimSpace(name) == "" {
return fmt.Errorf("profile file is required")
}
c.profileFS, c.profileFile = nil, name
return nil
}
}
// WithFallbackProfileFS selects lower-precedence profile fixture files.
func WithFallbackProfileFS(fsys fs.FS, root string) option {
return func(c *choices) error {
if err := validSourceFS(fsys, root); err != nil {
return err
}
c.fallbackFS, c.fallbackRoot = fsys, root
return nil
}
}
func validSourceFS(filesystem fs.FS, root string) error {
if filesystem == nil || strings.TrimSpace(root) == "" {
return fmt.Errorf("source filesystem and root are required")
}
return nil
}
// WithProfiles selects in-memory profile fixtures.
func WithProfiles(profiles ...Profile) option {
return func(c *choices) error { c.profiles, c.profilesSet = profiles, true; return nil }
}
// WithBackend adds one backend fixture.
func WithBackend(backend Backend) option {
if backend.QueueCapacity != nil {
capacity := *backend.QueueCapacity
backend.QueueCapacity = &capacity
}
return func(c *choices) error { c.backends = append(c.backends, backend); return nil }
}
// NewEngine expresses existing regression scenarios through ConfigKit sources.
func NewEngine(config Config, opts ...any) (*promptkit.Engine, error) {
var choices choices
var runtime []promptkit.Option
for _, candidate := range opts {
switch value := candidate.(type) {
case nil:
case option:
if err := value(&choices); err != nil {
return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err)
}
case promptkit.Option:
runtime = append(runtime, value)
default:
return nil, fmt.Errorf("%w: unsupported fixture option %T", promptkit.ErrInvalidConfig, candidate)
}
}
var underlays []configkit.Underlay
add := func(inputs []configkit.Input, err error) error {
if err != nil {
return err
}
if len(inputs) > 0 {
underlays = append(underlays, configkit.Underlay{Inputs: inputs})
}
return nil
}
if choices.fallbackFS != nil {
if err := add(sourceInputs("fallback profiles", choices.fallbackFS, choices.fallbackRoot, "profiles")); err != nil {
return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err)
}
}
if choices.profileFile != "" {
if err := add(singleInput(choices.profileFile, "profiles")); err != nil {
return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err)
}
} else if choices.profileFS != nil {
if err := add(sourceInputs("profiles", choices.profileFS, choices.profileRoot, "profiles")); err != nil {
return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err)
}
} else if config.ProfileDir != "" {
if err := add(directoryInputs(config.ProfileDir, "profiles")); err != nil {
return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err)
}
}
if choices.profilesSet {
profiles := make(map[string]promptkit.ProfileConfig, len(choices.profiles))
for _, value := range choices.profiles {
if _, exists := profiles[value.ID]; exists {
return nil, fmt.Errorf("%w: duplicate profile fixture %q", promptkit.ErrInvalidConfig, value.ID)
}
profiles[value.ID] = profileConfig(value)
}
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("memory profiles", profiles)}}})
}
for index, value := range choices.backends {
config := promptkit.BackendConfig{Endpoint: &value.Endpoint, APIKeyEnv: &value.APIKeyEnv,
ConcurrencyLimit: &value.ConcurrencyLimit, QueueCapacity: value.QueueCapacity, ExtraParams: value.ExtraParams}
underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{{Source: promptkit.BackendSource(fmt.Sprintf("backend %d", index), map[string]promptkit.BackendConfig{value.ID: config})}}})
}
if choices.promptFile != "" {
if err := add(singleInput(choices.promptFile, "prompts")); err != nil {
return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err)
}
} else if choices.promptFS != nil {
if err := add(sourceInputs("prompts", choices.promptFS, choices.promptRoot, "prompts")); err != nil {
return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err)
}
} else if config.PromptDir != "" {
if err := add(directoryInputs(config.PromptDir, "prompts")); err != nil {
return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err)
}
}
return promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: underlays},
SchemaDir: config.SchemaDir, Timeout: config.Timeout, HTTPClient: config.HTTPClient,
TransientRetries: config.TransientRetries}, runtime...)
}
func profileConfig(value Profile) promptkit.ProfileConfig {
config := promptkit.ProfileConfig{ExtraParams: value.ExtraParams}
if value.BaseProfileID != "" {
config.Extends = &value.BaseProfileID
}
if value.BackendID != "" {
config.Backend = &value.BackendID
}
if value.Endpoint != "" {
config.Endpoint = &value.Endpoint
}
if value.Model != "" {
config.Model = &value.Model
}
if value.Temperature != 0 {
config.Temperature = &value.Temperature
}
if value.TopP != 0 {
config.TopP = &value.TopP
}
if value.MaxTokens != 0 {
config.MaxTokens = &value.MaxTokens
}
if value.TimeoutSeconds != 0 {
config.TimeoutSeconds = &value.TimeoutSeconds
}
if value.ServiceTier != "" {
config.ServiceTier = &value.ServiceTier
}
if value.ReasoningEffort != "" {
config.ReasoningEffort = &value.ReasoningEffort
}
if value.APIKeyRequired {
config.APIKeyRequired = &value.APIKeyRequired
}
return config
}
func directoryInputs(directory, collection string) ([]configkit.Input, error) {
if _, err := os.Stat(directory); err != nil {
return nil, err
}
return sourceInputs(directory, os.DirFS(directory), ".", collection)
}
func sourceInputs(label string, filesystem fs.FS, root, collection string) ([]configkit.Input, error) {
return promptkit.FilesystemInputs(label, convertingFS{FS: filesystem, collection: collection}, root)
}
func singleInput(name, collection string) ([]configkit.Input, error) {
info, err := os.Stat(name)
if err != nil {
return nil, err
}
if info.IsDir() {
return nil, fmt.Errorf("source %s is a directory", name)
}
filesystem := convertingFS{FS: os.DirFS(filepath.Dir(name)), collection: collection}
namespace := configkit.NewFilesystem(name, filesystem)
return []configkit.Input{{Source: namespace.File(filepath.Base(name)), Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: "."}}}, nil
}
type convertingFS struct {
fs.FS
collection string
}
func (source convertingFS) Open(name string) (fs.File, error) {
file, err := source.FS.Open(name)
if err != nil || (!strings.HasSuffix(strings.TrimSpace(name), ".yaml") && !strings.HasSuffix(strings.TrimSpace(name), ".yml")) {
return file, err
}
info, err := file.Stat()
if err != nil {
file.Close()
return nil, err
}
data, err := io.ReadAll(file)
closeErr := file.Close()
if err != nil {
return nil, err
}
if closeErr != nil {
return nil, closeErr
}
converted := convertDocument(data, source.collection)
return &memoryFile{Reader: bytes.NewReader(converted), info: info}, nil
}
type memoryFile struct {
*bytes.Reader
info fs.FileInfo
}
func (file *memoryFile) Stat() (fs.FileInfo, error) { return file.info, nil }
func (file *memoryFile) Close() error { return nil }
func convertDocument(data []byte, collection string) []byte {
var document map[string]any
if err := yaml.Unmarshal(data, &document); err != nil || document == nil {
return data
}
if _, keyed := document[collection]; keyed {
return data
}
id, idOK := document["id"]
if !idOK {
return data
}
delete(document, "id")
if collection == "profiles" {
if base, exists := document["base_profile"]; exists {
delete(document, "base_profile")
document["extends"] = base
}
data, _ = yaml.Marshal(map[string]any{"profiles": map[string]any{fmt.Sprint(id): document}})
return data
}
version, versionOK := document["version"]
if !versionOK {
return data
}
delete(document, "version")
data, _ = yaml.Marshal(map[string]any{"prompts": map[string]any{fmt.Sprint(id): map[string]any{fmt.Sprint(version): document}}})
return data
}

25
transient_retries_contract_test.go
View File

@@ -11,7 +11,6 @@ import (
"testing"
"gitea.maximumdirect.net/eric/promptkit"
"gitea.maximumdirect.net/eric/promptkit/testfixture"
)
func retryHTTPResponse(status int, content string) *http.Response {
@@ -25,13 +24,13 @@ func retryHTTPResponse(status int, content string) *http.Response {
func retryTestEngine(t *testing.T, budget *int, next func(int) *http.Response) (*promptkit.Engine, *int) {
t.Helper()
calls := 0
config := contractConfig(frameworkSchemaDir)
config := simpleConfig("message")
config.TransientRetries = budget
config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) {
calls++
return next(calls), nil
})}
engine, err := testfixture.NewEngine(config)
engine, err := promptkit.NewEngine(config)
if err != nil {
t.Fatal(err)
}
@@ -45,7 +44,7 @@ func TestBuiltInRetryDefaultsAndRequestOverride(t *testing.T) {
}
return retryHTTPResponse(200, "ok")
})
result, err := engine.Run(context.Background(), generationErrorRunRequest())
result, err := engine.Run(context.Background(), simpleGenerationRequest())
if err != nil || result == nil || result.RawOutput != "ok" || *calls != 3 {
t.Fatalf("default result=%+v, error=%v, calls=%d", result, err, *calls)
}
@@ -54,7 +53,7 @@ func TestBuiltInRetryDefaultsAndRequestOverride(t *testing.T) {
engine, calls = retryTestEngine(t, &engineBudget, func(int) *http.Response { return retryHTTPResponse(503, "") })
engineBudget = 5 // Construction copied the configured value.
zero := 0
req := generationErrorRunRequest()
req := simpleGenerationRequest()
req.TransientRetries = &zero
_, err = engine.Run(context.Background(), req)
if !errors.Is(err, promptkit.ErrLLMGenerate) || errors.Is(err, promptkit.ErrRetriesExhausted) || *calls != 1 {
@@ -70,14 +69,14 @@ func TestBuiltInRetryDefaultsAndRequestOverride(t *testing.T) {
func TestRetryValidationAndPreparedFreezing(t *testing.T) {
negative := -1
config := contractConfig(frameworkSchemaDir)
config := simpleConfig("message")
config.TransientRetries = &negative
if _, err := testfixture.NewEngine(config); !errors.Is(err, promptkit.ErrInvalidConfig) {
if _, err := promptkit.NewEngine(config); !errors.Is(err, promptkit.ErrInvalidConfig) {
t.Fatalf("negative engine budget = %v", err)
}
engine, calls := retryTestEngine(t, nil, func(int) *http.Response { return retryHTTPResponse(503, "") })
req := generationErrorRunRequest()
req := simpleGenerationRequest()
req.TransientRetries = &negative
if _, err := engine.Prepare(context.Background(), req); !errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("Prepare negative budget = %v", err)
@@ -95,7 +94,7 @@ func TestRetryValidationAndPreparedFreezing(t *testing.T) {
largeBudgetEngine, largeCalls := retryTestEngine(t, &maximum, func(int) *http.Response {
return retryHTTPResponse(200, "ok")
})
if _, err := largeBudgetEngine.Run(context.Background(), generationErrorRunRequest()); err != nil || *largeCalls != 1 {
if _, err := largeBudgetEngine.Run(context.Background(), simpleGenerationRequest()); err != nil || *largeCalls != 1 {
t.Fatalf("large valid budget: error=%v, calls=%d", err, *largeCalls)
}
@@ -124,7 +123,7 @@ func TestBuiltInRetryBudgetAppliesToRepairGeneration(t *testing.T) {
return retryHTTPResponse(200, `{"ok":true}`)
}
})
req := generationErrorRunRequest()
req := simpleGenerationRequest()
req.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: 1}
result, err := engine.Run(context.Background(), req)
if err != nil || result == nil || !result.Validation.IsValid || *calls != 4 {
@@ -135,7 +134,7 @@ func TestBuiltInRetryBudgetAppliesToRepairGeneration(t *testing.T) {
func TestRetryExhaustionRetainsPublicProviderCause(t *testing.T) {
one := 1
engine, calls := retryTestEngine(t, &one, func(int) *http.Response { return retryHTTPResponse(503, "") })
result, err := engine.Run(context.Background(), generationErrorRunRequest())
result, err := engine.Run(context.Background(), simpleGenerationRequest())
if result != nil || *calls != 2 || !errors.Is(err, promptkit.ErrRetriesExhausted) || !errors.Is(err, promptkit.ErrLLMGenerate) {
t.Fatalf("exhaustion result=%+v, error=%v, calls=%d", result, err, *calls)
}
@@ -161,9 +160,9 @@ func TestRetryExhaustionRetainsPublicProviderCause(t *testing.T) {
func TestInjectedClientIsNotAutomaticallyRetried(t *testing.T) {
client := &fakeLLMClient{err: errors.New("injected failure")}
engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(client))
engine := newSimpleEngine(t, "message", promptkit.WithLLMClient(client))
count := 3
req := generationErrorRunRequest()
req := simpleGenerationRequest()
req.TransientRetries = &count
_, err := engine.Run(context.Background(), req)
if len(client.requests) != 1 || !errors.Is(err, client.err) || errors.Is(err, promptkit.ErrRetriesExhausted) {