diff --git a/appended_messages_contract_test.go b/appended_messages_contract_test.go index d3758b1..0503926 100644 --- a/appended_messages_contract_test.go +++ b/appended_messages_contract_test.go @@ -8,7 +8,6 @@ import ( "testing" "gitea.maximumdirect.net/eric/promptkit" - "gitea.maximumdirect.net/eric/promptkit/testfixture" ) func TestAppendedMessageRoleConstantsAreStrings(t *testing.T) { @@ -30,15 +29,7 @@ func TestAppendedMessagesComposeFrozenEffectivePrompt(t *testing.T) { {Role: promptkit.RoleUser, Content: "corrective request", CacheControl: cacheControl}, } client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "output"}} - engine, err := testfixture.NewEngine( - testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "configured prompt"), "."), - testfixture.WithProfiles(testfixture.Profile{ID: "profile", Endpoint: "http://example.test/v1", Model: "model"}), - promptkit.WithLLMClient(client), - ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } + engine := newSimpleEngine(t, "configured prompt", promptkit.WithLLMClient(client)) request := promptkit.RunRequest{PromptID: "prompt", AppendedMessages: appended} plain, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"}) diff --git a/capacity_contract_test.go b/capacity_contract_test.go index d275816..976e909 100644 --- a/capacity_contract_test.go +++ b/capacity_contract_test.go @@ -7,8 +7,8 @@ import ( "testing" "time" + "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/promptkit" - "gitea.maximumdirect.net/eric/promptkit/testfixture" ) func TestEngineLimitsInjectedClientConcurrency(t *testing.T) { @@ -151,17 +151,7 @@ func TestUnlimitedBackendsRetainInjectedClientConcurrency(t *testing.T) { { name: "endpoint-only profile", configure: func(t *testing.T, client promptkit.LLMClient) *promptkit.Engine { - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), - testfixture.WithProfiles(testfixture.Profile{ - ID: "profile", Endpoint: "http://endpoint.example/v1", Model: "model", - }), - promptkit.WithLLMClient(client), - ) - if err != nil { - t.Fatalf("construct endpoint-only engine: %v", err) - } - return engine + return newSimpleEngine(t, "message", promptkit.WithLLMClient(client)) }, }, } @@ -251,27 +241,35 @@ func newBackendCapacityEngine( reader promptkit.ArtifactReader, ) *promptkit.Engine { t.Helper() - promptFS := contractPromptFS("prompt", "profile", "message") + promptID, content := "prompt", "message" + var inputs []any if reader != nil { - promptFS = contractInputPromptFS() + promptID, content = "input-prompt", `{{input "input"}}` + inputs = []any{map[string]any{"name": "input", "required": true}} } - options := []any{ - testfixture.WithPromptFS(promptFS, "."), - testfixture.WithBackend(testfixture.Backend{ - ID: "limited", - Endpoint: "http://backend.example/v1", - ConcurrencyLimit: limit, - QueueCapacity: queueCapacity, - }), - testfixture.WithProfiles(testfixture.Profile{ - ID: "profile", BackendID: "limited", Model: "model", - }), - promptkit.WithLLMClient(client), + backend := map[string]any{"endpoint": "http://backend.example/v1", "concurrency_limit": limit} + if queueCapacity != nil { + backend["queue_capacity"] = *queueCapacity } + definition := map[string]any{ + "default_profile": "profile", + "messages": []any{map[string]any{"role": "user", "content": content}}, + "output": map[string]any{"format": "text", "validation_mode": "none"}, + } + if inputs != nil { + definition["inputs"] = inputs + } + options := []promptkit.Option{promptkit.WithLLMClient(client)} if reader != nil { options = append(options, promptkit.WithArtifactReader(reader)) } - engine, err := testfixture.NewEngine(testfixture.Config{}, options...) + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{ + Source: configkit.Mapping("capacity fixture", map[string]any{ + "backends": map[string]any{"limited": backend}, + "profiles": map[string]any{"profile": map[string]any{"backend": "limited", "model": "model"}}, + "prompts": map[string]any{promptID: map[string]any{"1": definition}}, + }), + }}}, options...) if err != nil { t.Fatalf("construct capacity engine: %v", err) } diff --git a/consumer_test_helpers_test.go b/consumer_test_helpers_test.go new file mode 100644 index 0000000..0597853 --- /dev/null +++ b/consumer_test_helpers_test.go @@ -0,0 +1,73 @@ +package promptkit_test + +import ( + "context" + "fmt" + "net/http" + "sync/atomic" + "testing" + + "gitea.maximumdirect.net/eric/ConfigKit" + "gitea.maximumdirect.net/eric/promptkit" +) + +type fakeLLMClient struct { + response *promptkit.GenerateResponse + responses []*promptkit.GenerateResponse + err error + requests []promptkit.GenerateRequest +} + +func (f *fakeLLMClient) Generate(_ context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + f.requests = append(f.requests, request) + if f.err != nil { + return nil, f.err + } + if len(f.responses) > 0 { + index := len(f.requests) - 1 + if index >= len(f.responses) { + return nil, fmt.Errorf("no response configured for generation %d", index+1) + } + return f.responses[index], nil + } + return f.response, nil +} + +type countingLLMClient struct{ calls *atomic.Int64 } + +func (client countingLLMClient) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + if client.calls != nil { + client.calls.Add(1) + } + return &promptkit.GenerateResponse{Content: "ok"}, nil +} + +type roundTripFunc func(*http.Request) (*http.Response, error) + +func (f roundTripFunc) RoundTrip(request *http.Request) (*http.Response, error) { return f(request) } + +// newSimpleEngine exercises the public constructor with a complete keyed +// document. Tests that need layering or resources select their own inputs. +func newSimpleEngine(t *testing.T, content string, options ...promptkit.Option) *promptkit.Engine { + t.Helper() + engine, err := promptkit.NewEngine(simpleConfig(content), options...) + if err != nil { + t.Fatalf("construct consumer test engine: %v", err) + } + return engine +} + +func simpleConfig(content string) promptkit.Config { + return promptkit.Config{Sources: promptkit.ConfigurationSources{ + Main: &configkit.Input{Source: configkit.Mapping("consumer test", map[string]any{ + "profiles": map[string]any{"profile": map[string]any{ + "endpoint": "http://example.test/v1", "model": "model", + }}, + "prompts": map[string]any{"prompt": map[string]any{"1": map[string]any{ + "default_profile": "profile", + "messages": []any{map[string]any{"role": "user", "content": content}}, + "output": map[string]any{"format": "text", "validation_mode": "none"}, + }}}, + })}, + }} +} diff --git a/docs/internal/overview.md b/docs/internal/overview.md index 832e2b0..87f21ec 100644 --- a/docs/internal/overview.md +++ b/docs/internal/overview.md @@ -11,29 +11,28 @@ contributor workflow and validation. | Component | Implemented responsibility | References | | --- | --- | --- | -| Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request, result, prompt-inspection, profile-inspection, and ordinary-run observation values, opaque prepared-execution handles, profile construction, extension interfaces, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go) | -| `examples/go-library/prepare` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, and `Prepare`. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) | -| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) | +| Root `promptkit` package | Provides the supported engine facade, ConfigKit source helpers, profile and backend mapping conveniences, injection options, public requests and results, explicit inspection, observation values, opaque prepared-execution handles, value conversion, redacted formatting, and typed capacity and generation error mapping. | [Package GoDoc](../../doc.go), [observations](../../observations.go), [prepared execution](../../prepared_execution.go), [backend API](../../backends.go), [engine facade](../../engine.go) | +| `examples/go-library/prepare` | Demonstrates offline preparation and explicit configuration inspection with a prompt file, programmatic profile, and inline input. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) | +| `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, programmatic profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) | | `examples/go-library/observe` | Demonstrates offline built-in HTTP retry and synchronous metadata observation across structured proposal and assessment runs using embedded sources, two profiles on one finite backend, and an application decision. It is not a public library package. | [Example program](../../examples/go-library/observe/main.go) | -| `internal/backend` | Constructs each engine's immutable registry from maintained definitions and consumer additions, validates and defensively copies definitions through the shared JSON-value package, and consumes the LLM-owned OpenAI-compatible reserved request-field rule. | [Backend registry](../../internal/backend/registry.go) | +| `internal/backend` | Holds a defensive, immutable lookup and capacity-policy index for final backend definitions validated by the configuration adapter. | [Backend registry](../../internal/backend/registry.go) | | `internal/catalog` | Selects released keyed catalog assets and PromptKit-owned profile definitions as ConfigKit underlays; effective validation belongs to the configuration loader. | [Catalog adapter](../../internal/catalog/catalog.go), [internal sources](sources.md#profiles-and-built-ins) | -| `internal/configuration` | Loads keyed backend, profile, and prompt configuration through ConfigKit; records effective field presence, validates final values, captures indexed prompt definitions and referenced message content, and constructs immutable backend and presence-aware profile snapshots. It is not yet connected to engine assembly. | [Typed loader](../../internal/configuration/load.go), [internal sources](sources.md#typed-configuration-loading) | +| `internal/configuration` | Loads keyed backend, profile, and prompt configuration through ConfigKit; records effective field presence, validates final values, captures indexed prompt definitions and referenced message content, constructs backend and presence-aware profile snapshots, and retains an explicit read-only inspection view. | [Typed loader](../../internal/configuration/load.go), [internal sources](sources.md#typed-configuration-loading) | | `internal/capacity` | Owns engine-local bounded execution admission and FIFO model-generation permits for limited backend IDs, including cancellation-safe waiter removal and client wrapping. | [Internal capacity management](capacity.md) | | `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation, and owns source-neutral invariants for shared execution settings, OpenAI-compatible base endpoints, session identifiers, and output contracts. Source parsing, required fields, other source-specific normalization, defaulting, and boundary-specific error classification remain with their callers. | [Domain declarations](../../internal/domain/domain.go), [endpoint invariant](../../internal/domain/endpoint.go) | -| `internal/engine` | Assembles external catalogs, Promptkit-owned profiles, repositories, registry, capacity, collaborators, repairer, and the one use-case runner for each engine instance. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#profiles-and-built-ins), [internal runner](runner.md) | +| `internal/engine` | Assembles selected catalog and caller configuration into one snapshot, then constructs capacity, rendering, validation, generation, repair, and the use-case runner for each engine. | [Engine assembly](../../internal/engine/engine.go), [internal sources](sources.md#profiles-and-built-ins), [internal runner](runner.md) | | `internal/generation` | Coordinates frozen built-in HTTP attempts and one-shot injected calls through active backend permits, including bounded status retries, cancellable backoff, and attempt observations. | [Internal generation coordination](generation.md) | | `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) | | `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) | | `internal/jsonvalue` | Validates and deeply copies bounded JSON-compatible extra-parameter and prepared-schema trees while preserving supported concrete value types and rejecting cycles or excessive depth and work. | [JSON values](../../internal/jsonvalue/jsonvalue.go) | -| `internal/legacycatalog` | Temporarily supports the existing public engine construction path with the previously published flat catalogs until engine assembly switches to ConfigKit. | [Existing catalog adapter](../../internal/legacycatalog/catalog.go), [internal sources](sources.md#profiles-and-built-ins) | -| `internal/promptdef` | Loads strictly decoded, validated prompt definitions from filesystem and `fs.FS` sources, including version selection and contained file-backed message content. | [Framework formats](../formats.md), [prompt-definition repository](../../internal/promptdef/filesystem_repository.go) | -| `internal/profile` | Owns immutable in-memory and filesystem profile repositories, local profile validation, raw-source overlays with error-preserving fallback, and inherited-profile resolution. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go), [internal sources](sources.md#profiles-and-built-ins) | +| `internal/promptdef` | Provides a narrow captured-prompt lookup contract, not-found identity, and execution-ready definition hash. | [Prompt lookup](../../internal/promptdef/repository.go), [framework formats](../formats.md) | +| `internal/profile` | Provides a narrow captured-profile lookup contract and not-found identity. | [Profile lookup](../../internal/profile/repository.go), [framework formats](../formats.md) | | `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) | | `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) | | `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources and creates operation-local validation plans with canonical contained schema resources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) | | `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including bounded structured non-success response decoding, successful-response decoding, authentication, deadline handling, and ownership of the OpenAI-compatible reserved request-field policy. | [Internal model client](llm.md) | | `internal/observation` | Owns invocation-local run IDs, presence-aware usage and phase accounting, and synchronous metadata event delivery for ordinary and claimed prepared runs and generation attempts. | [Internal execution observation](observation.md) | -| `internal/usecase` | Resolves prompt definitions and hashes, profiles, backends, and targets for exact inspection and request settings for preparation, and coordinates ordinary and single-claim prepared execution across internal sources, rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) | +| `internal/usecase` | Selects captured prompts and profiles, applies request overrides and completion checks, and coordinates ordinary and single-claim prepared execution across rendering, artifact loading, operation-local validation plans, generation, capacity, and bounded repair. | [Internal runner](runner.md), [prepared-execution implementation](../../internal/usecase/prepared_execution.go) | The internal engine assembles these internal components without exposing their representations. Consumers depend only on the root facade. diff --git a/docs/internal/runner.md b/docs/internal/runner.md index f592830..26e6eab 100644 --- a/docs/internal/runner.md +++ b/docs/internal/runner.md @@ -4,7 +4,7 @@ This document describes Promptkit's implemented internal orchestration. The [architecture policy](../policy/architecture.md) owns dependency and consumer -boundaries. The [source and validation document](sources.md) owns repository, +boundaries. The [source and validation document](sources.md) owns configuration, artifact, rendering, and validation behavior, while the [model-client document](llm.md) owns generation behavior and failure categories. @@ -16,11 +16,10 @@ and override semantics consumed by the runner. ## Collaborators -`Runner` coordinates narrow internal interfaces for prompt definitions, -profiles, backend resolution, artifacts, rendering, model generation, and -validation. The internal engine supplies one immutable registry containing the -built-in backend and validated consumer additions, one engine-local run -admitter, and a generation coordinator using the same capacity manager. Validation +`Runner` coordinates narrow internal interfaces for captured prompt definitions +and resolved profiles, artifacts, rendering, model generation, and validation. +The internal engine supplies one engine-local run admitter and a generation +coordinator using the same capacity manager. Validation plans and provider-facing schema metadata come from the validator's preparation interface. The internal engine supplies one default output repairer through the explicit @@ -35,11 +34,11 @@ The runner has no durable run or session store. The runner uses one prompt-selection and hashing boundary for ordinary preparation and exact prompt inspection. Preparation retains its early -request-ID check before direct-session normalization; both operations then use -the configured prompt repository to select one definition, load referenced -message content, and calculate the same prompt hash. +request-ID check before direct-session normalization; both operations select +one captured definition and its construction-time hash. Referenced message +content and static template syntax were captured when the engine was built. -Inspection stops after that structural lookup. It does not parse templates or +Inspection stops after that structural lookup. It does not render templates or touch profile, artifact, schema, renderer, validator, admission, or model collaborators. The root [`Engine.InspectPrompt`](../../engine.go) GoDoc owns the public operation's exact contract. @@ -49,20 +48,11 @@ the public operation's exact contract. The runner uses one profile-selection and target-resolution boundary for ordinary preparation and exact profile inspection. Preparation first selects a request profile or a prompt default; inspection begins with its required -explicit profile ID. Both then apply the ordinary source precedence, resolve a -named backend, and construct the effective target from framework, backend, and -profile values. - -The private ConfigKit loader can instead supply an already resolved, -presence-aware target to this boundary. Request overrides still apply afterward, -and preparation checks endpoint and model completeness only after those -overrides. This path remains internal until the engine assembly switches to the -loader; the existing public engine still uses the repository path above. - -For the existing repository path, inspection validates the resulting endpoint -and model. For the precomputed target path it can report an incomplete profile; -preparation remains responsible for checking completeness after request -overrides. Neither path checks credential availability or performs prompt, +explicit profile ID. Both use the captured, presence-aware target constructed +from the final effective profile and backend configuration. Request overrides +apply afterward; preparation checks endpoint and model completeness only after +those overrides. Inspection can report a partial profile without requiring a +runnable target. Neither path checks credential availability or performs prompt, artifact, schema, rendering, admission, or model-client work. The root [`Engine.InspectProfile`](../../engine.go) GoDoc owns the public operation's exact contract. @@ -74,11 +64,10 @@ where a run can be assigned to its selected backend pool. The resolution phase performs only the work needed to validate routing and admission: 1. validate the required prompt selection and normalize any direct session ID; -2. load the prompt definition and hash the original definition; +2. select the captured prompt definition and hash; 3. select the request profile or the prompt's default profile; -4. resolve the profile's backend ID, when present; -5. resolve application-neutral defaults, backend defaults, profile values, - and explicit request overrides in that order; +4. retain the profile's selected backend identity, when present; +5. apply explicit request overrides to its captured backend-completed target; 6. validate endpoint, model, numeric overrides, and credential requirements; 7. resolve and validate the effective output contract without loading its schema; and @@ -110,26 +99,26 @@ Pointer-based numeric overrides preserve an explicit zero. Invalid negative or out-of-range values fail as invalid requests. Endpoint overrides do not change the selected backend identity. Non-empty extra-parameter maps replace whole lower-precedence maps. A direct API key takes precedence over environment -lookup; otherwise request, profile, and backend environment-variable names -apply in that order. A profile requiring a direct key clears an inherited +lookup; otherwise request, explicit profile, and backend environment-variable +names apply in that order. A profile requiring a direct key suppresses an inherited backend environment name unless the request supplies its own name. Secret values remain excluded from serialized metadata. Reasoning overrides are tri-state: nil inherits the profile, a pointer to a nonblank string trims and replaces it, and a pointer to a blank string clears -it. A nonblank direct session is normalized before source loading, bypasses +it. A nonblank direct session is normalized before prompt selection, bypasses the prompt session template, and is applied after ordinary message rendering. A blank direct value retains prompt-template behavior. The runner clears the template only on a value copy of the definition, so the definition hash always -describes the original source while the rendered-prompt hash includes the +describes captured effective content while the rendered-prompt hash includes the effective direct or rendered session and the complete effective message sequence. The rendered-prompt hash uses a versioned, length-framed SHA-256 encoding of that session, every role and content value, and cache-control presence and values; its hexadecimal value is opaque. -The registry is read-only after engine construction. Concurrent `Prepare` and -`Run` calls resolve independent defensive backend values and keep all -invocation state local. +The configuration snapshot is read-only after engine construction. Concurrent +`Prepare` and `Run` calls copy their resolved targets and keep all invocation +state local. ## Run Flow @@ -223,7 +212,7 @@ generation, validation, repair, and cancellation failures. Other context cancellation propagates through the invoked collaborator and is classified by the owning operation. In particular, cancellation observed by validation retains the context identity through the validation error category. -An overlong direct session is an invalid request before source loading, while +An overlong direct session is an invalid request before prompt selection, while an invalid or overlong prompt session template remains a prompt-render failure. An unknown selected backend, or a selected backend with no configured resolver, is classified as a profile-load failure. diff --git a/docs/internal/sources.md b/docs/internal/sources.md index 25deefa..3861dfd 100644 --- a/docs/internal/sources.md +++ b/docs/internal/sources.md @@ -2,141 +2,68 @@ ## Purpose -This document describes Promptkit's implemented internal source, artifact, +This document describes Promptkit's implemented configuration, artifact, rendering, and output-validation behavior. The -[architecture policy](../policy/architecture.md) owns the library boundary and -dependency rules. None of these internal packages is a supported consumer API, -and the internal engine assembles them behind the public facade's source options and values. -The [framework format reference](../formats.md) owns the exact file fields, -validation modes, built-in catalog, and source precedence. +[architecture policy](../policy/architecture.md) owns dependency rules, while +the [framework format reference](../formats.md) owns fields, defaults, and +source semantics. Internal packages are not consumer APIs. ## Typed Configuration Loading -`internal/configuration` now contains a private ConfigKit adapter for keyed -backend, profile, and prompt mappings. It performs one layered load, records -effective field presence (including explicit null), normalizes backend and -profile fields through ConfigKit resolution, and validates final backend and -profile domain values. A lower-layer domain value replaced before the handoff -is not separately validated. It also validates effective prompt fields, reads -referenced message content using the contributing field's passive source -context, checks static template syntax, and indexes immutable definitions by -exact ID and version. The loader also builds an immutable backend registry from -the final definitions and indexes profile execution targets after inheritance, -applying the selected backend's defaults only then. These targets retain -explicit numeric-zero presence and null suppression for runtime execution. -Its focused tests exercise real ConfigKit loading. +`internal/configuration` is the single adapter from ConfigKit's final +effective configuration to execution-owned definitions. It performs one +layered load, captures effective presence (including explicit null), records +normalization and defaults through resolution, and validates all effective +backend, profile, and prompt values. Replaced lower-layer domain values are +not separately validated. It builds an immutable backend index and +presence-aware profile targets, applying final backend defaults only after +profile inheritance. It retains ConfigKit's result privately for explicit +inspection, not for per-run resolution. The runtime/domain packages do not +depend on ConfigKit. -This loader is not yet wired into engine assembly. The public engine continues -to use the source and repository behavior described below. The public -`DirectoryInputs` and `FilesystemInputs` helpers select sorted YAML inputs for -one strict-composition group and attach an explicit resource root; they do not -assign precedence or enable imports. +The root `DirectoryInputs` and `FilesystemInputs` helpers discover sorted YAML +files under an explicit root, provide resource context, and leave imports +disabled. Each returned slice belongs to one strict-composition group; file +order does not confer priority. The engine prepends selected catalog and +Promptkit-owned definitions, then caller underlays and optional main. The +[configuration tests](../../internal/configuration/load_test.go) exercise the +real loader and final-value handoff. ## Prompt Definitions -`internal/promptdef` uses one source-neutral flow for prompt selection and -normalization. That flow scans normalized YAML ID and version metadata, -requires one strictly decoded document per file, classifies errors for the -selected definition, detects duplicates, and normalizes the exact match. -Small operating-system and `fs.FS` adapters own discovery, byte reads, display -paths, content opening, and root containment. Each lookup remains a -point-in-time scan: definitions and catalogs are not cached, and file-backed -message content is opened only for the exact selected candidate. +The configuration adapter indexes each prompt by exact ID and version, +materializes file-backed message content using the exact effective field's +origin and resource context, and checks static message/session template syntax +at construction. It stores captured content, compiled templates, declared +input metadata, and a definition hash. It closes content-file handles before +returning and retains no source capability for public inspection. -Message roles are normalized through the shared domain owner by trimming -Unicode whitespace and lowercasing. Only `developer`, `system`, `user`, and -`assistant` are published; invalid roles remain selected prompt-definition -failures rather than becoming request errors. Cache-control metadata uses the -same shared domain normalization and defensive-copy rule. - -Operating-system sources enforce containment against canonical roots and -targets so symlinks cannot escape. Injected `fs.FS` sources enforce containment -in their clean relative path namespace. A single-file source uses the selected -prompt file's containing directory as its root. Every content path must be -relative and is opened from its exact parsed text after a separate blank check; -contained parent components and whitespace-bearing names remain valid. - -Exact prompt inspection performs one point-in-time lookup through that same -repository and validates referenced message content before returning declared -metadata. It does not parse templates or read profile, input, or schema -sources, and it does not retain the definition for a later execution. - -Its package tests own prompt selection, strict decoding, definition validation, -duplicate detection, and source containment: -[prompt-definition repository tests](../../internal/promptdef/repository_test.go). +The runner selects from this index for preparation and `InspectPrompt`. +Neither path rescans configuration or reopens message files. `InspectPrompt` +returns declared metadata without rendering or resolving profile, input, or +schema resources. Runtime rendering still checks request variables and input +helpers. The [prompt adapter tests](../../internal/configuration/prompts_test.go) +cover resource origins, context containment, capture, and selection. ## Profiles And Built-Ins -`internal/profile` owns immutable in-memory and filesystem repositories, local -validation, overlays, and resolution for execution profiles. In-memory -definitions are normalized and validated once at that repository boundary and -are defensively copied on storage and lookup. A file contains -exactly one YAML document and its trimmed YAML `id` is its only selection -identity; filenames do not confer authority. Each point lookup reads discovered -files once for their metadata and reuses the selected file's bytes for strict -decoding; unrelated profiles are not fully decoded. Strict selected decoding -recognizes `base_profile` and the optional `backend` field, trims their values, -and permits inherited target fields only when a base is named. File-backed -`extra_params` values are validated and defensively copied through the shared -bounded JSON-value owner before a profile is published. OpenAI-compatible -reserved-field policy remains with the model-client and backend-registry owners. - -`LoadFSRepository` is the eager immutable loading boundary for internal -catalog consumers. It discovers and strictly validates every raw profile once, -preserves safe source metadata including explicitly present YAML fields, and -publishes independently copied values from memory. It does not resolve profile -inheritance. Configured consumer sources continue to use the lazy point lookup -repositories described above. - `internal/catalog` selects the released keyed OpenRouter and Rakestrawhome assets as one ConfigKit underlay and PromptKit-owned profiles as the next. It discovers YAML inputs and supplies passive filesystem context without parsing or pre-validating catalog definitions. The configuration loader validates only the final effective backend and profile values, including -consumer overlays of maintained bases or backends. The new path has no catalog -source-layout, ownership-prefix, connection-field, or secret-key rejection. +consumer overlays of maintained bases or backends. There is no catalog-only +preflight, reserved source namespace, or replay of backend registration. -Until public engine assembly switches to that loader, `internal/legacycatalog` -supports the existing public construction path with the published flat `/v2` -assets. The repository behavior below describes that temporary path. - -The overlay repository consults the next repository only when the -higher-precedence repository reports that a profile is absent. A reliably -selected malformed profile stops fallback, while an unrelated malformed file -does not become authoritative through its filename. Loading does not check -backend registry membership because the available registry belongs to the -assembled engine; the runner checks membership during preparation and exact -profile inspection. - -The internal engine assembles one raw composite catalog in precedence order: -in-memory profiles, one ordinary configured source, an application fallback -source, Promptkit-owned profiles, then the maintained external catalog. It -constructs the owned definitions through the in-memory repository and validates -their maintained-only resolution and backend membership before consumer sources -are overlaid. An explicit file or `fs.FS` profile source replaces -`Config.ProfileDir` within the ordinary configured-source category. One outer -resolving repository wraps that complete raw catalog, so each base lookup -observes the same precedence and shadowing rules. - -The resolving repository traverses every selected chain afresh, retains no -cache, detects cycles, limits a chain to 32 profiles, merges root-to-leaf into a -new caller-owned value, and validates the final target before publishing it. It -does not check backend registry membership. Exact `base_profile` syntax, merge -rules, and consumer-visible failure behavior belong to the [framework format -reference](../formats.md#profile-inheritance). - -Exact profile inspection performs one point-in-time resolved lookup through -those profile sources and checks the final target without reading prompt, input, -or schema sources. It does not retain that lookup for a later execution. -Prepared execution instead freezes the fully resolved target; a later ordinary -operation performs a fresh traversal. - -The imported external catalog provides external profiles and matching backend -definitions. The internal engine owns the Promptkit profiles documented in the -[framework format reference](../formats.md#built-in-profile-catalog). Profile -loading and overlay behavior are owned by the [profile repository tests](../../internal/profile/repository_test.go), while -catalog completeness, backend selection, and duplicate IDs are owned by the -[catalog adapter tests](../../internal/catalog/catalog_test.go). +ConfigKit resolves profile inheritance before the handoff. The configuration +adapter validates effective references and field values, then builds target +snapshots with backend defaults and presence/suppression state. A profile can +remain partial until runtime overrides complete it. `InspectProfile` selects +that captured target without prompt, artifact, schema, credential lookup, or +provider work. Preparation applies runtime overrides and then checks target +completeness. The [target tests](../../internal/configuration/targets_test.go) +cover effective target interpretation, while the [catalog tests](../../internal/catalog/catalog_test.go) +cover selected released assets. ## Ordinary Artifacts @@ -161,14 +88,15 @@ implemented reader behavior and failures. ## Rendering -`internal/prompt` renders definition messages as Go templates using named -artifacts and variables. Within one render, each referenced artifact body is -converted to text lazily and cached by input name for reuse across the session -and every message; the cache is not shared across renders. Conversion uses -bounded chunks and preserves the artifact bytes exactly. +`internal/prompt` renders captured, pre-parsed definition templates using named +artifacts and variables. It clones templates to bind request-specific input +helpers without mutating shared state. Within one render, each referenced +artifact body is converted to text lazily and cached by input name for reuse +across the session and every message; the cache is not shared across renders. +Conversion uses bounded chunks and preserves the artifact bytes exactly. -Session and message parsing and execution remain synchronous. The renderer -checks cancellation before and after each parse and execution boundary, +Template parsing occurs during engine construction. Runtime execution remains +synchronous. The renderer checks cancellation around execution boundaries, between artifact conversion chunks, around each message, and before publishing the complete prompt. It cannot interrupt template work already in progress and never publishes a partial prompt after observing cancellation. It validates and @@ -205,9 +133,10 @@ not detach dependency work into background goroutines. `Prepare` discards its validation plan after returning metadata. `Run` retains its plan for initial and repaired-output validation, then discards it with the operation. `PrepareExecution` retains the plan in its private frozen payload; -`RunPrepared` uses that plan without reopening prompt, profile, input, or -schema sources or rerendering the request. A later ordinary `Run` always -performs fresh source resolution and preparation. +`RunPrepared` uses that plan without reopening configuration, message, input, +or schema resources or rerendering the request. A later ordinary `Run` uses +the same captured configuration but prepares request artifacts and schemas +afresh. The [validator tests](../../internal/validate/standard_validator_test.go) own basic, JSON, JSON Schema, source resolution, schema loading, compilation, diff --git a/docs/oss/attribution.md b/docs/oss/attribution.md index 06de135..452cf5f 100644 --- a/docs/oss/attribution.md +++ b/docs/oss/attribution.md @@ -30,21 +30,6 @@ License and notices: [LGPL 3.0](../../LICENSE) | [Source](https://gitea.maximumdirect.net/eric/ConfigKit/src/commit/dc09959cca4e105f49ec316c6d4ba585bd2340bd). -## Promptkit OpenRouter Catalog v2 - -Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2` -at `v2.20260926.0`. - -Copyright (C) 2026 Eric Rakestraw - -SPDX-License-Identifier: MIT - -License and notices: [MIT](licenses/MIT.txt) | -[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/4f731d217e0bc39151c010f50fb5912706e7b6b3/COPYRIGHT) | -[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/4f731d217e0bc39151c010f50fb5912706e7b6b3/LICENSE). - -[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/4f731d217e0bc39151c010f50fb5912706e7b6b3). - ## Promptkit OpenRouter Catalog v3 Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3` @@ -60,21 +45,6 @@ License and notices: [MIT](licenses/MIT.txt) | [Source](https://gitea.maximumdirect.net/eric/promptkit-backend-openrouter/src/commit/f17d208cb7ad1fb043cda2266cd3011d11b40aa0). -## Promptkit Rakestrawhome Catalog v2 - -Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2` -at `v2.20260926.0`. - -Copyright (C) 2026 Eric Rakestraw - -SPDX-License-Identifier: MIT - -License and notices: [MIT](licenses/MIT.txt) | -[COPYRIGHT](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/48918a3ad39fd99843cda94a86133b1b3881d596/COPYRIGHT) | -[LICENSE](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/48918a3ad39fd99843cda94a86133b1b3881d596/LICENSE). - -[Source](https://gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/src/commit/48918a3ad39fd99843cda94a86133b1b3881d596). - ## Promptkit Rakestrawhome Catalog v3 Selected module: `gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3` @@ -117,7 +87,7 @@ License and notices: [BSD 3-Clause](licenses/BSD-3-Clause.txt) | ## Go YAML v3 -Selected module: `gopkg.in/yaml.v3` at `v3.0.1`. +Selected module: `gopkg.in/yaml.v3` at `v3.0.1` (transitive dependency). Copyright (c) 2006-2010 Kirill Simonov diff --git a/docs/policy/architecture.md b/docs/policy/architecture.md index 0c15c03..768ce32 100644 --- a/docs/policy/architecture.md +++ b/docs/policy/architecture.md @@ -51,23 +51,22 @@ The [internal component overview](../internal/overview.md) owns the inventory of implemented packages and their concrete responsibilities. This policy owns the boundaries and dependency rules those components must preserve. -The internal engine assembles one immutable backend registry, one capacity manager, -the internal repositories, renderer, validator, outbound client, and use-case -runner. The registry contains validated maintained definitions plus engine-scoped -consumer additions. The root facade translates public values and errors at the -library boundary. -The internal engine constructs the capacity manager from the registry's immutable -policy snapshot, wraps the selected built-in or injected model client, and -supplies bounded admission to the runner. The defaults and renderer depend on -the domain model. Prompt-definition and profile repositories use the domain -model, file catalog, and YAML decoder. The catalog adapter validates imported -immutable maintained data, including its reserved namespace, before internal -engine assembly supplies it to the profile package. Artifact reading uses the -domain model and application-neutral defaults. Validation uses the domain model, -file catalog, and JSON Schema implementation. The model client uses the domain -model and application-neutral defaults, and an injected or standard-library HTTP -client. The use-case runner depends on the narrow interfaces owned by each -internal component, including backend lookup and run admission. +The internal engine composes selected catalog and caller ConfigKit inputs once, +then assembles the captured definitions, capacity manager, renderer, validator, +outbound client, and use-case runner. The configuration adapter interprets and +validates only the final effective values, including presence, null, and +resource context. It retains source history for explicit operator inspection, +not for validation or a second resolution pass. The root facade translates +public values and errors at the library boundary. + +The engine constructs capacity policy from final backend definitions, wraps +the selected built-in or injected model client, and supplies bounded admission +to the runner. Runtime/domain components consume execution-owned values, not +ConfigKit types. Artifact reading and validation use the domain model and +application-neutral defaults; validation also uses file-catalog helpers and +JSON Schema. The model client uses the domain model, defaults, and an injected +or standard-library HTTP client. The runner coordinates narrow lookup and +collaborator interfaces without rereading configuration. The current implementation follows this dependency direction: @@ -84,9 +83,10 @@ downstream consumers, including Scriptorium narrow injected abstractions ``` -The backend registry depends on the domain model and shared JSON-value -validation, has no mutation API after construction, and consumes the -OpenAI-compatible reserved request-field rule owned by the model client. The +The backend index depends on the domain model and shared JSON-value ownership +and has no mutation API after construction. The configuration adapter applies +the OpenAI-compatible reserved request-field rule owned by the model client +when it validates final provider maps. The capacity component depends on the domain model and the narrow internal model-client boundary, not on provider transport implementation. The model client does not depend on registry or capacity configuration. The facade diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index d355a6e..130c751 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -570,6 +570,8 @@ published, and no speculative release-note version is invented. ## Stage 11 — Remove Superseded Machinery And Consolidate Tests +**Completed:** 2026-09-30. + Scope: dead loading/resolution code and its obsolete tests, module cleanup, and internal architectural documentation. diff --git a/engine_test.go b/engine_test.go deleted file mode 100644 index 858a1b9..0000000 --- a/engine_test.go +++ /dev/null @@ -1,3598 +0,0 @@ -package promptkit_test - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "io" - "io/fs" - "math" - "net/http" - "net/http/httptest" - "os" - "path" - "path/filepath" - "reflect" - "strconv" - "strings" - "sync" - "testing" - "testing/fstest" - "time" - - "gitea.maximumdirect.net/eric/promptkit" - "gitea.maximumdirect.net/eric/promptkit/testfixture" -) - -const ( - frameworkContractRoot = "./testdata/framework" - frameworkPromptDir = frameworkContractRoot + "/prompts" - frameworkProfileDir = frameworkContractRoot + "/profiles" - frameworkSchemaDir = frameworkContractRoot + "/schemas" - - frameworkMarkdownSummaryPromptID = "contract.markdown_summary" - frameworkStructuredEventsPromptID = "contract.structured_events" - frameworkFastProfileID = "contract-fast" - frameworkQualityProfileID = "contract-quality" - - frameworkTranscriptPath = frameworkContractRoot + "/fixtures/transcript.md" - frameworkGlossaryPath = frameworkContractRoot + "/fixtures/glossary.yml" -) - -func TestNewEngineAllowsNoPrompts(t *testing.T) { - engine, err := promptkit.NewEngine(promptkit.Config{}) - if err != nil { - t.Fatalf("construct engine without prompts: %v", err) - } - if _, err := engine.InspectProfile(context.Background(), "mistral-small-3"); err != nil { - t.Fatalf("inspect maintained profile without prompts: %v", err) - } -} - -func TestNewEngineAcceptsMissingProfileDir(t *testing.T) { - _, err := testfixture.NewEngine(testfixture.Config{PromptDir: frameworkPromptDir}) - if err != nil { - t.Fatalf("expected missing profile dir to use built-ins, got %v", err) - } -} - -func TestPrepareWorksWithFrameworkContractCorpus(t *testing.T) { - engine, err := testfixture.NewEngine(testfixture.Config{ - PromptDir: frameworkPromptDir, - ProfileDir: frameworkProfileDir, - SchemaDir: frameworkSchemaDir, - }) - if err != nil { - t.Fatalf("construct engine from framework contract corpus: %v", err) - } - - tests := []struct { - name string - promptID string - profileID string - model string - structured bool - }{ - { - name: "markdown summary", - promptID: frameworkMarkdownSummaryPromptID, - profileID: frameworkFastProfileID, - model: "contract-fast-model", - }, - { - name: "structured events", - promptID: frameworkStructuredEventsPromptID, - profileID: frameworkQualityProfileID, - model: "contract-quality-model", - structured: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: tt.promptID, - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.File(frameworkTranscriptPath), - "glossary": promptkit.File(frameworkGlossaryPath), - }, - }) - if err != nil { - t.Fatalf("prepare framework contract prompt: %v", err) - } - if prepared.PromptID != tt.promptID { - t.Fatalf("unexpected prompt id: got %q, want %q", prepared.PromptID, tt.promptID) - } - if prepared.SelectedProfileID != tt.profileID { - t.Fatalf("unexpected selected profile: got %q, want %q", prepared.SelectedProfileID, tt.profileID) - } - if prepared.EffectiveModelParams.Model != tt.model { - t.Fatalf("unexpected effective model: got %q, want %q", prepared.EffectiveModelParams.Model, tt.model) - } - if len(prepared.Messages) != 2 { - t.Fatalf("expected rendered messages, got %d", len(prepared.Messages)) - } - if !strings.Contains(prepared.Messages[1].Content, "Nia labels the archive.") { - t.Fatalf("expected relative prompt content to render the transcript, got %q", prepared.Messages[1].Content) - } - if prepared.InputHashes["transcript"] == "" || prepared.InputHashes["glossary"] == "" { - t.Fatalf("expected input hashes, got %#v", prepared.InputHashes) - } - - if !tt.structured { - if prepared.StructuredOutput != nil { - t.Fatalf("expected no structured output specification, got %#v", prepared.StructuredOutput) - } - return - } - - if prepared.StructuredOutput == nil || prepared.StructuredOutput.JSONSchema == nil { - t.Fatalf("expected loaded JSON Schema structured output, got %#v", prepared.StructuredOutput) - } - schema, ok := prepared.StructuredOutput.JSONSchema.Schema.(map[string]any) - if !ok || schema["type"] != "object" { - t.Fatalf("expected loaded object JSON Schema, got %#v", prepared.StructuredOutput.JSONSchema.Schema) - } - properties, ok := schema["properties"].(map[string]any) - if !ok || properties["events"] == nil { - t.Fatalf("expected loaded events schema property, got %#v", schema) - } - }) - } -} - -func TestPrepareWorksWithInlineInputs(t *testing.T) { - engine := newContractEngine(t) - - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin scouts the tower.\nKara lights a lantern."), - "glossary": promptkit.InlineWithURI("memory://glossary.yml", "party:\n - Rin\n - Kara\n"), - }, - }) - if err != nil { - t.Fatalf("expected prepare to succeed, got %v", err) - } - if len(prepared.Messages) != 2 { - t.Fatalf("expected rendered messages, got %d", len(prepared.Messages)) - } - rendered := prepared.Messages[1].Content - if !strings.Contains(rendered, "Rin scouts the tower.") || !strings.Contains(rendered, "party:") { - t.Fatalf("expected inline inputs in rendered prompt, got %q", rendered) - } -} - -func TestPreparedRunJSONDoesNotExposeSecretOrTargetPresence(t *testing.T) { - const envName = "PROMPTKIT_API_KEY" - const secret = "public-api-test-secret" - t.Setenv(envName, secret) - - profileDir := t.TempDir() - writePublicProfileFileWithAPIKeyEnv(t, profileDir, "prepared-secret", "http://localhost:8000/v1", "prepared-secret-model", envName) - engine, err := testfixture.NewEngine(testfixture.Config{ - PromptDir: frameworkPromptDir, - ProfileDir: profileDir, - SchemaDir: frameworkSchemaDir, - }) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: frameworkStructuredEventsPromptID, - ProfileID: "prepared-secret", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.File(frameworkTranscriptPath), - "glossary": promptkit.File(frameworkGlossaryPath), - }, - }) - if err != nil { - t.Fatalf("expected prepare to succeed, got %v", err) - } - - payload, err := json.Marshal(prepared) - if err != nil { - t.Fatalf("expected prepared run to marshal, got %v", err) - } - out := string(payload) - if strings.Contains(out, secret) { - t.Fatalf("prepared run JSON leaked raw API key value: %s", out) - } - if !strings.Contains(out, envName) { - t.Fatalf("prepared run JSON should retain api_key_env name, got %s", out) - } - for _, forbidden := range []string{"TargetPresence", "target_presence"} { - if strings.Contains(out, forbidden) { - t.Fatalf("prepared run JSON exposed internal target presence metadata %q: %s", forbidden, out) - } - } -} - -func TestRunRequestFormattingRedactsDirectAPIKey(t *testing.T) { - const ( - secret = "run-request-api-key-sentinel" - inputURI = "memory://run-request-uri-sentinel" - inputBody = "run-request-body-sentinel" - variableValue = "run-request-variable-sentinel" - ) - req := promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: frameworkFastProfileID, - APIKey: secret, - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.InlineWithURI(inputURI, inputBody), - }, - Vars: map[string]string{"audience": variableValue}, - AppendedMessages: []promptkit.RenderedMessage{{ - Role: "private-role-sentinel", - Content: "private-appended-content-sentinel", - CacheControl: &promptkit.CacheControl{ - Type: "private-cache-sentinel", - TTL: "private-ttl-sentinel", - }, - }}, - } - - for _, formatted := range []string{ - req.String(), - req.GoString(), - fmt.Sprintf("%v", req), - fmt.Sprintf("%+v", req), - fmt.Sprintf("%#v", req), - } { - for _, privateValue := range []string{secret, inputURI, inputBody, variableValue, "private-role-sentinel", "private-appended-content-sentinel", "private-cache-sentinel", "private-ttl-sentinel"} { - if strings.Contains(formatted, privateValue) { - t.Fatalf("formatted RunRequest leaked private value %q: %s", privateValue, formatted) - } - } - for _, summary := range []string{ - `PromptID:"` + frameworkMarkdownSummaryPromptID + `"`, - `ProfileID:"` + frameworkFastProfileID + `"`, - "APIKeySet:true", - "Inputs:1", - "Vars:1", - "AppendedMessages:1", - } { - if !strings.Contains(formatted, summary) { - t.Fatalf("formatted RunRequest omitted structural summary %q: %s", summary, formatted) - } - } - } - - payload, err := json.Marshal(req) - if err != nil { - t.Fatalf("expected RunRequest to marshal, got %v", err) - } - if strings.Contains(string(payload), secret) { - t.Fatalf("RunRequest JSON leaked API key: %s", payload) - } -} - -func TestGenerateRequestFormattingRedactsDirectAPIKey(t *testing.T) { - const secret = "generate-request-secret" - req := promptkit.GenerateRequest{ - Prompt: promptkit.RenderedPrompt{Messages: []promptkit.RenderedMessage{ - {Role: "user", Content: "secret prompt content"}, - }}, - Target: promptkit.ExecutionTarget{ - Model: "test-model", - ExtraParams: map[string]any{ - "provider_option": "on", - }, - }, - APIKey: secret, - } - - for _, formatted := range []string{ - fmt.Sprint(req), - fmt.Sprintf("%+v", req), - fmt.Sprintf("%#v", req), - } { - if strings.Contains(formatted, secret) { - t.Fatalf("formatted GenerateRequest leaked API key: %s", formatted) - } - if strings.Contains(formatted, "secret prompt content") { - t.Fatalf("formatted GenerateRequest leaked prompt content: %s", formatted) - } - if !strings.Contains(formatted, "APIKeySet:true") { - t.Fatalf("formatted GenerateRequest should indicate an API key is set, got %s", formatted) - } - } - - payload, err := json.Marshal(req) - if err != nil { - t.Fatalf("expected GenerateRequest to marshal, got %v", err) - } - if strings.Contains(string(payload), secret) { - t.Fatalf("GenerateRequest JSON leaked API key: %s", payload) - } -} - -func TestEngineExecutionSettingPrecedence(t *testing.T) { - floatPointer := func(value float64) *float64 { - return &value - } - intPointer := func(value int) *int { - return &value - } - stringPointer := func(value string) *string { - return &value - } - - defaultsProfile := executionProfileFixture{ - id: "settings-defaults", - endpoint: "http://profile-defaults.test/v1", - model: "profile-defaults-model", - serviceTier: "profile-defaults-tier", - reasoningEffort: "profile-defaults-reasoning", - apiKeyEnv: "PROMPTKIT_PRECEDENCE_DEFAULTS", - extraParamSource: "profile-defaults", - } - profileSettings := executionProfileFixture{ - id: "settings-profile", - endpoint: "http://profile-settings.test/v1", - model: "profile-settings-model", - temperature: 0.31, - maxTokens: 311, - topP: 0.61, - timeoutSeconds: 71, - serviceTier: "profile-settings-tier", - reasoningEffort: "profile-settings-reasoning", - apiKeyEnv: "PROMPTKIT_PRECEDENCE_PROFILE", - extraParamSource: "profile-settings", - } - requestProfile := executionProfileFixture{ - id: "settings-request", - endpoint: "http://profile-request.test/v1", - model: "profile-request-model", - temperature: 0.29, - maxTokens: 299, - topP: 0.59, - timeoutSeconds: 79, - serviceTier: "profile-request-tier", - reasoningEffort: "profile-request-reasoning", - apiKeyEnv: "PROMPTKIT_PRECEDENCE_REQUEST_PROFILE", - extraParamSource: "profile-request", - } - zeroOverrideProfile := executionProfileFixture{ - id: "settings-zero", - endpoint: "http://profile-zero.test/v1", - model: "profile-zero-model", - temperature: 0.43, - maxTokens: 433, - topP: 0.73, - timeoutSeconds: 83, - serviceTier: "profile-zero-tier", - reasoningEffort: "profile-zero-reasoning", - apiKeyEnv: "PROMPTKIT_PRECEDENCE_ZERO", - extraParamSource: "profile-zero", - } - - requestTarget := promptkit.ExecutionTarget{ - Endpoint: "http://request-settings.test/v1", - Model: "request-settings-model", - Temperature: 0.87, - MaxTokens: 877, - TopP: 0.97, - TimeoutSeconds: 177, - ServiceTier: "request-settings-tier", - ReasoningEffort: "request-settings-reasoning", - APIKeyEnv: "PROMPTKIT_PRECEDENCE_REQUEST", - ExtraParams: map[string]any{"source": "request-settings"}, - } - zeroOverrideTarget := executionTargetFromProfileFixture(zeroOverrideProfile) - zeroOverrideTarget.Temperature = 0 - zeroOverrideTarget.MaxTokens = 0 - zeroOverrideTarget.TopP = 0 - zeroOverrideTarget.TimeoutSeconds = 0 - - tests := []struct { - name string - profile executionProfileFixture - override *promptkit.ExecutionTargetOverride - want promptkit.ExecutionTarget - wantPresence promptkit.ExecutionTargetPresence - }{ - { - name: "explicit zero provider controls disable deadline", - profile: defaultsProfile, - wantPresence: promptkit.ExecutionTargetPresence{Temperature: true, MaxTokens: true, TopP: true, TimeoutSeconds: true}, - want: promptkit.ExecutionTarget{ - Endpoint: defaultsProfile.endpoint, - Model: defaultsProfile.model, - Temperature: 0, - MaxTokens: 0, - TopP: 0, - TimeoutSeconds: 0, - ServiceTier: defaultsProfile.serviceTier, - ReasoningEffort: defaultsProfile.reasoningEffort, - APIKeyEnv: defaultsProfile.apiKeyEnv, - ExtraParams: map[string]any{"source": defaultsProfile.extraParamSource}, - }, - }, - { - name: "profile settings replace framework defaults", - profile: profileSettings, - want: executionTargetFromProfileFixture(profileSettings), - wantPresence: promptkit.ExecutionTargetPresence{Temperature: true, MaxTokens: true, TopP: true, TimeoutSeconds: true}, - }, - { - name: "request settings replace profile settings", - profile: requestProfile, - override: &promptkit.ExecutionTargetOverride{ - Endpoint: requestTarget.Endpoint, - Model: requestTarget.Model, - Temperature: floatPointer(requestTarget.Temperature), - MaxTokens: intPointer(requestTarget.MaxTokens), - TopP: floatPointer(requestTarget.TopP), - TimeoutSeconds: intPointer(requestTarget.TimeoutSeconds), - ServiceTier: requestTarget.ServiceTier, - ReasoningEffort: stringPointer(requestTarget.ReasoningEffort), - APIKeyEnv: requestTarget.APIKeyEnv, - ExtraParams: requestTarget.ExtraParams, - }, - want: requestTarget, - wantPresence: promptkit.ExecutionTargetPresence{Temperature: true, MaxTokens: true, TopP: true, TimeoutSeconds: true}, - }, - { - name: "explicit request zero replaces profile settings", - profile: zeroOverrideProfile, - override: &promptkit.ExecutionTargetOverride{ - Temperature: floatPointer(0), - MaxTokens: intPointer(0), - TopP: floatPointer(0), - TimeoutSeconds: intPointer(0), - }, - want: zeroOverrideTarget, - wantPresence: promptkit.ExecutionTargetPresence{Temperature: true, MaxTokens: true, TopP: true, TimeoutSeconds: true}, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - t.Setenv(tt.profile.apiKeyEnv, "set") - if tt.override != nil && tt.override.APIKeyEnv != "" { - t.Setenv(tt.override.APIKeyEnv, "set") - } - - profileDir := t.TempDir() - writeExecutionProfileFixture(t, profileDir, tt.profile) - fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}} - engine, err := testfixture.NewEngine(testfixture.Config{ - PromptDir: frameworkPromptDir, - ProfileDir: profileDir, - SchemaDir: frameworkSchemaDir, - }, promptkit.WithLLMClient(fake)) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - - _, err = engine.Run(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: tt.profile.id, - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Nia labels the archive."), - "glossary": promptkit.Inline("archive: A catalogued collection."), - }, - Execution: tt.override, - }) - if err != nil { - t.Fatalf("run engine: %v", err) - } - if len(fake.requests) != 1 { - t.Fatalf("expected one generation request, got %d", len(fake.requests)) - } - got := fake.requests[0] - if !reflect.DeepEqual(got.Target, tt.want) { - t.Fatalf("unexpected effective target:\ngot=%#v\nwant=%#v", got.Target, tt.want) - } - if got.TargetPresence != tt.wantPresence { - t.Fatalf("unexpected target presence: got=%+v want=%+v", got.TargetPresence, tt.wantPresence) - } - }) - } - - t.Run("blank request reasoning clears profile setting", func(t *testing.T) { - profile := executionProfileFixture{ - id: "settings-reasoning-clear", - endpoint: "http://profile-reasoning.test/v1", - model: "profile-reasoning-model", - reasoningEffort: "medium", - } - profileDir := t.TempDir() - writeExecutionProfileFixture(t, profileDir, profile) - engine, err := testfixture.NewEngine(testfixture.Config{ - PromptDir: frameworkPromptDir, - ProfileDir: profileDir, - SchemaDir: frameworkSchemaDir, - }) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: profile.id, - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Nia labels the archive."), - "glossary": promptkit.Inline("archive: A catalogued collection."), - }, - Execution: &promptkit.ExecutionTargetOverride{ - ReasoningEffort: stringPointer(" \t "), - }, - }) - if err != nil { - t.Fatalf("prepare engine: %v", err) - } - if prepared.EffectiveModelParams.ReasoningEffort != "" { - t.Fatalf("expected blank request reasoning to clear profile value, got %q", prepared.EffectiveModelParams.ReasoningEffort) - } - }) -} - -func TestRunSucceedsWithInjectedLLMClient(t *testing.T) { - const envName = "PROMPTKIT_API_KEY" - const secret = "run-secret-value" - t.Setenv(envName, secret) - - fake := &fakeLLMClient{ - response: &promptkit.GenerateResponse{ - Content: "# Summary\n\nDone.", - Usage: promptkit.TokenUsage{ - PromptTokens: 10, - CompletionTokens: 5, - TotalTokens: 15, - CachedTokens: 3, - CacheWriteTokens: 2, - }, - }, - } - engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(fake)) - - result, err := engine.Run(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - Execution: &promptkit.ExecutionTargetOverride{ - APIKeyEnv: envName, - }, - }) - if err != nil { - t.Fatalf("expected run to succeed, got %v", err) - } - if result.RunID == "" { - t.Fatalf("expected run id") - } - if result.RawOutput != fake.response.Content { - t.Fatalf("unexpected raw output: %q", result.RawOutput) - } - if string(result.Artifact.Body) != fake.response.Content { - t.Fatalf("unexpected artifact body: %q", string(result.Artifact.Body)) - } - if result.Artifact.ContentType != "text/markdown" { - t.Fatalf("unexpected artifact content type: %q", result.Artifact.ContentType) - } - if result.Validation.Status != promptkit.ValidationPassed || !result.Validation.IsValid { - t.Fatalf("expected passed validation, got %+v", result.Validation) - } - if result.PromptID != frameworkMarkdownSummaryPromptID || result.SelectedProfileID != frameworkFastProfileID || result.ModelName != "contract-fast-model" { - t.Fatalf("unexpected run metadata: %+v", result) - } - if result.Usage.TotalTokens != 15 || result.Usage.CachedTokens != 3 || result.Usage.CacheWriteTokens != 2 { - t.Fatalf("unexpected usage: %+v", result.Usage) - } - - payload, err := json.Marshal(result) - if err != nil { - t.Fatalf("expected run result to marshal, got %v", err) - } - if strings.Contains(string(payload), secret) { - t.Fatalf("run result JSON leaked raw API key value: %s", payload) - } -} - -func TestEngineRunWithDirectorySourcesAndFileInputs(t *testing.T) { - fake := &fakeLLMClient{ - response: &promptkit.GenerateResponse{ - Content: `{"events":[{"title":"Archive labelled"}]}`, - Usage: promptkit.TokenUsage{ - PromptTokens: 42, - CompletionTokens: 36, - TotalTokens: 78, - }, - }, - } - engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(fake)) - - result, err := engine.Run(context.Background(), promptkit.RunRequest{ - PromptID: frameworkStructuredEventsPromptID, - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.File(frameworkTranscriptPath), - "glossary": promptkit.File(frameworkGlossaryPath), - }, - }) - if err != nil { - t.Fatalf("expected run to succeed, got %v", err) - } - if result.PromptID != frameworkStructuredEventsPromptID || result.SelectedProfileID != frameworkQualityProfileID { - t.Fatalf("unexpected run metadata: %+v", result) - } - if result.RunID == "" || result.PromptHash == "" || result.RenderedPromptHash == "" { - t.Fatalf("expected run and prompt hashes, got %+v", result) - } - if result.InputHashes["transcript"] == "" || result.InputHashes["glossary"] == "" { - t.Fatalf("expected both input hashes, got %#v", result.InputHashes) - } - if len(fake.requests) != 1 || fake.requests[0].StructuredOutput == nil || - fake.requests[0].StructuredOutput.Type != promptkit.StructuredOutputJSONSchema || - fake.requests[0].StructuredOutput.JSONSchema == nil || - fake.requests[0].StructuredOutput.JSONSchema.Schema == nil { - t.Fatalf("expected provider JSON Schema structured output, got %+v", fake.requests) - } - if result.Validation.Status != promptkit.ValidationPassed || !result.Validation.IsValid || result.Validation.Mode != promptkit.ValidationJSONSchema { - t.Fatalf("expected passed JSON Schema validation, got %+v", result.Validation) - } - if result.Artifact.ContentType != "application/json" { - t.Fatalf("expected JSON artifact, got %q", result.Artifact.ContentType) - } - if result.RawOutput != fake.response.Content || result.Usage != fake.response.Usage { - t.Fatalf("expected preserved output and usage, got output=%q usage=%+v", result.RawOutput, result.Usage) - } - if result.StartTime.IsZero() || result.EndTime.IsZero() || result.EndTime.Before(result.StartTime) || result.Duration < 0 { - t.Fatalf("expected ordered non-zero timestamps and non-negative duration, got start=%v end=%v duration=%v", result.StartTime, result.EndTime, result.Duration) - } -} - -func TestRunPassesPreparedRequestToInjectedLLMClient(t *testing.T) { - const directKey = "direct-injected-key" - const directSession = "assembled-session" - fake := &fakeLLMClient{ - response: &promptkit.GenerateResponse{Content: `{"events":[{"title":"Archive labelled"}]}`}, - } - engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(fake)) - - runRequest := promptkit.RunRequest{ - PromptID: frameworkStructuredEventsPromptID, - SessionID: " " + directSession + " ", - APIKey: directKey, - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - } - prepared, err := engine.Prepare(context.Background(), runRequest) - if err != nil { - t.Fatalf("expected prepare to succeed, got %v", err) - } - result, err := engine.Run(context.Background(), runRequest) - if err != nil { - t.Fatalf("expected run to succeed, got %v", err) - } - if len(fake.requests) != 1 { - t.Fatalf("expected one generate request, got %d", len(fake.requests)) - } - req := fake.requests[0] - if len(req.Prompt.Messages) != 2 || !strings.Contains(req.Prompt.Messages[1].Content, "Rin opens the gate.") { - t.Fatalf("expected rendered prompt in generate request, got %+v", req.Prompt) - } - if prepared.SessionID != directSession || - req.Prompt.SessionID != directSession || - result.SessionID != directSession { - t.Fatalf( - "direct session did not propagate consistently: prepared=%q generated=%q result=%q", - prepared.SessionID, - req.Prompt.SessionID, - result.SessionID, - ) - } - if req.StructuredOutput == nil || req.StructuredOutput.Type != promptkit.StructuredOutputJSONSchema || req.StructuredOutput.JSONSchema == nil { - t.Fatalf("expected structured output handoff, got %+v", req.StructuredOutput) - } - if req.APIKey != directKey { - t.Fatalf("expected direct key on injected generate request") - } - payload, err := json.Marshal(req) - if err != nil { - t.Fatalf("expected generate request to marshal, got %v", err) - } - if strings.Contains(string(payload), directKey) { - t.Fatalf("generate request JSON leaked direct API key: %s", payload) - } -} - -func TestEngineRunPropagatesCallerCancellation(t *testing.T) { - const synchronizationTimeout = 5 * time.Second - - started := make(chan struct{}) - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - close(started) - <-req.Context().Done() - return nil, req.Context().Err() - }) - engine, err := testfixture.NewEngine(testfixture.Config{ - PromptDir: frameworkPromptDir, - ProfileDir: frameworkProfileDir, - SchemaDir: frameworkSchemaDir, - HTTPClient: &http.Client{Transport: transport}, - }) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - result := make(chan error, 1) - go func() { - _, err := engine.Run(ctx, promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Nia labels the archive."), - "glossary": promptkit.Inline("archive: A catalogued collection."), - }, - }) - result <- err - }() - - watchdog := time.NewTimer(synchronizationTimeout) - defer watchdog.Stop() - select { - case <-started: - case err := <-result: - t.Fatalf("Engine.Run returned before the transport started: %v", err) - case <-watchdog.C: - t.Fatal("timed out waiting for the transport to start") - } - - cancel() - select { - case err := <-result: - if !errors.Is(err, promptkit.ErrLLMGenerate) { - t.Fatalf("expected ErrLLMGenerate after caller cancellation, got %v", err) - } - if !errors.Is(err, context.Canceled) { - t.Fatalf("expected context cancellation identity, got %v", err) - } - case <-watchdog.C: - t.Fatal("timed out waiting for Engine.Run to return after cancellation") - } -} - -func TestRunRejectsReservedExtraParamsBeforeProviderCall(t *testing.T) { - called := false - transport := roundTripFunc(func(*http.Request) (*http.Response, error) { - called = true - return nil, errors.New("provider should not be called") - }) - engine, err := testfixture.NewEngine(testfixture.Config{ - PromptDir: frameworkPromptDir, - ProfileDir: frameworkProfileDir, - SchemaDir: frameworkSchemaDir, - HTTPClient: &http.Client{Transport: transport}, - }) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - - _, err = engine.Run(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Nia labels the archive."), - "glossary": promptkit.Inline("archive: A catalogued collection."), - }, - Execution: &promptkit.ExecutionTargetOverride{ - ExtraParams: map[string]any{"model": "collision"}, - }, - }) - if !errors.Is(err, promptkit.ErrInvalidRequest) { - t.Fatalf("expected ErrInvalidRequest, got %v", err) - } - if called { - t.Fatal("expected reserved provider parameter to fail before the provider call") - } -} - -func TestRunUsesDirectAPIKeyWithDefaultLLMClient(t *testing.T) { - const directKey = "direct-public-key" - const missingEnv = "PROMPTKIT_PUBLIC_DIRECT_MISSING" - t.Setenv(missingEnv, "") - - var gotAuth string - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - gotAuth = r.Header.Get("Authorization") - if r.URL.Path != "/v1/chat/completions" { - t.Errorf("unexpected path: %s", r.URL.Path) - } - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{ - "choices": [{"message": {"role": "assistant", "content": "# Summary\n\nDone."}}], - "usage": {"prompt_tokens": 3, "completion_tokens": 4, "total_tokens": 7} -}`)) - })) - defer server.Close() - - profileDir := t.TempDir() - writePublicProfileFileWithAPIKeyEnv(t, profileDir, "direct-auth", server.URL+"/v1", "test-model", missingEnv) - engine, err := testfixture.NewEngine(testfixture.Config{ - PromptDir: frameworkPromptDir, - ProfileDir: profileDir, - SchemaDir: frameworkSchemaDir, - }) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - result, err := engine.Run(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "direct-auth", - APIKey: directKey, - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - }) - if err != nil { - t.Fatalf("expected run with direct API key to succeed, got %v", err) - } - if gotAuth != "Bearer "+directKey { - t.Fatalf("unexpected Authorization header: %q", gotAuth) - } - if result.Usage.TotalTokens != 7 { - t.Fatalf("unexpected usage: %+v", result.Usage) - } - - payload, err := json.Marshal(result) - if err != nil { - t.Fatalf("expected run result to marshal, got %v", err) - } - if strings.Contains(string(payload), directKey) { - t.Fatalf("run result JSON leaked direct API key: %s", payload) - } -} - -func TestRunUsesResolvedBackendWithBuiltInLLMClient(t *testing.T) { - const ( - backendID = "local-test" - envName = "PROMPTKIT_BACKEND_TRANSPORT_KEY" - apiKey = "synthetic-backend-key" - ) - t.Setenv(envName, apiKey) - - var ( - gotAuth string - gotBody map[string]any - ) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - gotAuth = r.Header.Get("Authorization") - if r.URL.Path != "/v1/chat/completions" { - t.Errorf("unexpected path: %s", r.URL.Path) - } - if err := json.NewDecoder(r.Body).Decode(&gotBody); err != nil { - t.Errorf("decode request body: %v", err) - } - w.Header().Set("Content-Type", "application/json") - _, _ = w.Write([]byte(`{ - "choices": [{"message": {"role": "assistant", "content": "# Summary\n\nDone."}}], - "usage": {"prompt_tokens": 3, "completion_tokens": 4, "total_tokens": 7} -}`)) - })) - defer server.Close() - - engine, err := testfixture.NewEngine(testfixture.Config{ - PromptDir: frameworkPromptDir, - SchemaDir: frameworkSchemaDir, - }, - testfixture.WithBackend(testfixture.Backend{ - ID: backendID, - Endpoint: server.URL + "/v1", - APIKeyEnv: envName, - ExtraParams: map[string]any{ - "provider": "synthetic", - }, - }), - testfixture.WithProfiles(testfixture.Profile{ - ID: "backend-transport", - BackendID: backendID, - Model: "test-model", - }), - ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - - result, err := engine.Run(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "backend-transport", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - }) - if err != nil { - t.Fatalf("run with resolved backend: %v", err) - } - if gotAuth != "Bearer "+apiKey { - t.Fatalf("unexpected Authorization header: %q", gotAuth) - } - if gotBody["model"] != "test-model" || gotBody["provider"] != "synthetic" { - t.Fatalf("backend defaults did not reach provider payload: %#v", gotBody) - } - for _, field := range []string{"backend_id", "api_key_env"} { - if _, ok := gotBody[field]; ok { - t.Fatalf("internal metadata field %q was serialized to provider payload: %#v", field, gotBody) - } - } - bodyJSON, err := json.Marshal(gotBody) - if err != nil { - t.Fatalf("marshal captured provider payload: %v", err) - } - if strings.Contains(string(bodyJSON), apiKey) { - t.Fatalf("credential value was serialized to provider payload: %s", bodyJSON) - } - if result.SelectedBackendID != backendID || - result.EffectiveModelParams.Endpoint != server.URL+"/v1" || - result.EffectiveModelParams.APIKeyEnv != envName { - t.Fatalf("unexpected resolved backend metadata: %+v", result) - } -} - -func TestPrepareDirectAPIKeyBypassesMissingEnvWithoutLeakingOrHashing(t *testing.T) { - const missingEnv = "PROMPTKIT_PUBLIC_PREPARE_MISSING" - const firstKey = "first-direct-key" - const secondKey = "second-direct-key" - t.Setenv(missingEnv, "") - - profileDir := t.TempDir() - writePublicProfileFileWithAPIKeyEnv(t, profileDir, "direct-prepare", "http://localhost:8000/v1", "test-model", missingEnv) - engine, err := testfixture.NewEngine(testfixture.Config{ - PromptDir: frameworkPromptDir, - ProfileDir: profileDir, - SchemaDir: frameworkSchemaDir, - }) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - baseReq := promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "direct-prepare", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - } - firstReq := baseReq - firstReq.APIKey = firstKey - firstPrepared, err := engine.Prepare(context.Background(), firstReq) - if err != nil { - t.Fatalf("expected prepare with direct API key to succeed, got %v", err) - } - secondReq := baseReq - secondReq.APIKey = secondKey - secondPrepared, err := engine.Prepare(context.Background(), secondReq) - if err != nil { - t.Fatalf("expected prepare with alternate direct API key to succeed, got %v", err) - } - - if firstPrepared.PromptHash != secondPrepared.PromptHash { - t.Fatalf("direct API keys changed prompt hash: %q vs %q", firstPrepared.PromptHash, secondPrepared.PromptHash) - } - if firstPrepared.RenderedPromptHash != secondPrepared.RenderedPromptHash { - t.Fatalf("direct API keys changed rendered prompt hash: %q vs %q", firstPrepared.RenderedPromptHash, secondPrepared.RenderedPromptHash) - } - - payload, err := json.Marshal(firstPrepared) - if err != nil { - t.Fatalf("expected prepared run to marshal, got %v", err) - } - if strings.Contains(string(payload), firstKey) { - t.Fatalf("prepared run JSON leaked direct API key: %s", payload) - } -} - -func TestOptionalMissingCredentialsReachUpstream(t *testing.T) { - const missingEnv = "PROMPTKIT_PUBLIC_AUTH_MISSING" - const providerBody = `{"error":{"message":"authentication failed","type":"authentication_error","code":"invalid_api_key"}}` - t.Setenv(missingEnv, "") - - called := false - config := testfixture.Config{ - PromptDir: frameworkPromptDir, - SchemaDir: frameworkSchemaDir, - HTTPClient: &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { - called = true - if values := req.Header.Values("Authorization"); len(values) != 0 { - t.Fatalf("Authorization values = %q, want absent", values) - } - return &http.Response{ - StatusCode: http.StatusUnauthorized, - ContentLength: int64(len(providerBody)), - Body: io.NopCloser(strings.NewReader(providerBody)), - }, nil - })}, - } - engine, err := testfixture.NewEngine(config, - testfixture.WithBackend(testfixture.Backend{ - ID: "optional-auth", - Endpoint: "http://provider.test/v1", - APIKeyEnv: missingEnv, - }), - testfixture.WithProfiles(testfixture.Profile{ - ID: "optional-auth-profile", - BackendID: "optional-auth", - Model: "test-model", - }), - ) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - result, err := engine.Run(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "optional-auth-profile", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - }) - if !called { - t.Fatal("optional missing credential did not reach upstream") - } - if result != nil { - t.Fatalf("result = %+v, want nil", result) - } - if errors.Is(err, promptkit.ErrInvalidRequest) || errors.Is(err, promptkit.ErrAPIKeyEnvMissing) { - t.Fatalf("error = %v, want upstream generation error without credential identities", err) - } - if !errors.Is(err, promptkit.ErrLLMGenerate) { - t.Fatalf("error = %v, want ErrLLMGenerate", err) - } - var generationErr *promptkit.GenerationError - if !errors.As(err, &generationErr) { - t.Fatalf("error = %v, want GenerationError", err) - } - if generationErr.StatusCode() != http.StatusUnauthorized || - generationErr.ProviderType() != "authentication_error" || - generationErr.ProviderCode() != "invalid_api_key" || - generationErr.ProviderMessage() != "authentication failed" { - t.Fatalf("GenerationError = %+v, want structured upstream authentication failure", generationErr) - } -} - -func TestWithArtifactReaderRejectsNilReader(t *testing.T) { - _, err := testfixture.NewEngine(contractConfig(frameworkSchemaDir), promptkit.WithArtifactReader(nil)) - if !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("expected ErrInvalidConfig, got %v", err) - } -} - -func TestArtifactReaderReceivesPublicReferenceAndPreparesArtifact(t *testing.T) { - reader := &recordingArtifactReader{ - artifact: &promptkit.Artifact{ - ContentType: "text/plain", - Body: []byte("Reader-supplied transcript."), - URI: "reader://transcript", - Size: int64(len("Reader-supplied transcript.")), - Hash: "reader-transcript-hash", - }, - } - engine := newArtifactReaderEngine(t, reader) - - ref := promptkit.ArtifactRef{ - Type: promptkit.ArtifactRefInline, - URI: "reader://transcript", - Body: "request body", - } - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: "artifact-reader", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": ref, - }, - }) - if err != nil { - t.Fatalf("prepare with artifact reader: %v", err) - } - if len(reader.refs) != 1 || !reflect.DeepEqual(reader.refs[0], ref) { - t.Fatalf("reader received %#v, want %#v", reader.refs, ref) - } - if prepared.InputHashes["transcript"] != "reader-transcript-hash" { - t.Fatalf("unexpected input hash: %#v", prepared.InputHashes) - } - if len(prepared.Messages) != 1 || !strings.Contains(prepared.Messages[0].Content, "Reader-supplied transcript.") { - t.Fatalf("prepared prompt omitted reader artifact: %#v", prepared.Messages) - } -} - -func TestPrepareAcceptsExplicitEmptyInlineInputs(t *testing.T) { - promptSource := fstest.MapFS{ - "prompt.yaml": &fstest.MapFile{Data: []byte(`id: empty-inline -version: "1" -default_profile: profile -inputs: - - name: transcript - required: true -session_id: 'session-{{input "transcript"}}' -messages: - - role: user - content: 'before{{input "transcript"}}after' -output: - format: text - validation_mode: none -`)}, - } - engine, err := testfixture.NewEngine( - testfixture.Config{}, - testfixture.WithPromptFS(promptSource, "."), - testfixture.WithProfiles(testfixture.Profile{ - ID: "profile", Endpoint: "http://example.test/v1", Model: "model", - }), - ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - - for name, ref := range map[string]promptkit.ArtifactRef{ - "without URI": promptkit.Inline(""), - "with URI": promptkit.InlineWithURI("memory://empty", ""), - } { - t.Run(name, func(t *testing.T) { - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: "empty-inline", - Inputs: map[string]promptkit.ArtifactRef{"transcript": ref}, - }) - if err != nil { - t.Fatalf("prepare empty input: %v", err) - } - if prepared.SessionID != "session-" { - t.Fatalf("session ID = %q, want session-", prepared.SessionID) - } - if len(prepared.Messages) != 1 || prepared.Messages[0].Content != "beforeafter" { - t.Fatalf("messages = %#v, want empty input rendered between markers", prepared.Messages) - } - if prepared.InputHashes["transcript"] == "" { - t.Fatalf("empty input did not retain an equality value: %#v", prepared.InputHashes) - } - }) - } -} - -func TestArtifactReaderFailuresPreserveArtifactLoadErrors(t *testing.T) { - readerErr := errors.New("artifact reader failed") - - tests := []struct { - name string - ctx context.Context - reader *recordingArtifactReader - wantNested error - }{ - { - name: "reader error", - ctx: context.Background(), - reader: &recordingArtifactReader{err: readerErr}, - wantNested: readerErr, - }, - { - name: "nil artifact", - ctx: context.Background(), - reader: &recordingArtifactReader{}, - }, - { - name: "reader cancellation", - ctx: context.Background(), - reader: &recordingArtifactReader{read: func(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) { - return nil, context.Canceled - }}, - wantNested: context.Canceled, - }, - { - name: "public reader error", - ctx: context.Background(), - reader: &recordingArtifactReader{err: promptkit.ErrInvalidRequest}, - wantNested: promptkit.ErrInvalidRequest, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - engine := newArtifactReaderEngine(t, tc.reader) - _, err := engine.Prepare(tc.ctx, promptkit.RunRequest{ - PromptID: "artifact-reader", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("input"), - }, - }) - if !errors.Is(err, promptkit.ErrArtifactLoad) { - t.Fatalf("expected ErrArtifactLoad, got %v", err) - } - if tc.wantNested != nil && !errors.Is(err, tc.wantNested) { - t.Fatalf("expected nested %v, got %v", tc.wantNested, err) - } - }) - } -} - -func TestRunAddsLLMGenerateToCollaboratorPublicError(t *testing.T) { - injectedErr := errors.New("injected model client failure") - engine := newContractEngineWithOptions(t, frameworkSchemaDir, - promptkit.WithLLMClient(&fakeLLMClient{err: injectedErr}), - ) - - _, err := engine.Run(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - }) - if !errors.Is(err, promptkit.ErrLLMGenerate) { - t.Fatalf("expected ErrLLMGenerate, got %v", err) - } - if !errors.Is(err, injectedErr) { - t.Fatalf("expected preserved injected error, got %v", err) - } - var generationErr *promptkit.GenerationError - if errors.As(err, &generationErr) { - t.Fatalf("injected error became GenerationError: %v", err) - } -} - -func TestPrepareWithoutProfileMatchesSpecificPublicError(t *testing.T) { - promptDir := t.TempDir() - writePublicPromptFile(t, promptDir, "profile-required", "") - engine, err := testfixture.NewEngine(testfixture.Config{ - PromptDir: promptDir, - SchemaDir: frameworkSchemaDir, - }) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - - _, err = engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: "profile-required", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("input"), - }, - }) - if !errors.Is(err, promptkit.ErrInvalidRequest) { - t.Fatalf("expected ErrInvalidRequest, got %v", err) - } - if !errors.Is(err, promptkit.ErrProfileRequired) { - t.Fatalf("expected ErrProfileRequired, got %v", err) - } -} - -func TestRunValidationFailureReturnsResult(t *testing.T) { - fake := &fakeLLMClient{ - response: &promptkit.GenerateResponse{Content: ""}, - } - engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(fake)) - - result, err := engine.Run(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - }) - if err != nil { - t.Fatalf("expected validation failure as successful result, got %v", err) - } - if result.Validation.Status != promptkit.ValidationFailed || result.Validation.IsValid { - t.Fatalf("expected failed validation result, got %+v", result.Validation) - } - if len(result.Validation.Errors) == 0 { - t.Fatalf("expected validation errors") - } -} - -func TestPublicErrorsSupportErrorsIs(t *testing.T) { - llmErr := errors.New("llm failed") - - tests := []struct { - name string - req promptkit.RunRequest - client promptkit.LLMClient - schemaDir string - want error - notWant error - }{ - { - name: "invalid request", - req: promptkit.RunRequest{}, - client: &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}, - want: promptkit.ErrInvalidRequest, - }, - { - name: "prompt not found", - req: promptkit.RunRequest{PromptID: "missing.prompt"}, - client: &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}, - want: promptkit.ErrPromptNotFound, - notWant: promptkit.ErrPromptLoad, - }, - { - name: "profile not found", - req: promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "missing-profile", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - }, - }, - client: &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}, - want: promptkit.ErrProfileNotFound, - notWant: promptkit.ErrProfileLoad, - }, - { - name: "artifact load", - req: promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.File(filepath.Join(t.TempDir(), "does-not-exist.md")), - }, - }, - client: &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}, - want: promptkit.ErrArtifactLoad, - }, - { - name: "prompt render", - req: promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - }, - client: &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}, - want: promptkit.ErrPromptRender, - }, - { - name: "llm failure", - req: promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - }, - client: &fakeLLMClient{err: llmErr}, - want: promptkit.ErrLLMGenerate, - }, - { - name: "nil llm response", - req: promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - }, - client: &fakeLLMClient{}, - want: promptkit.ErrLLMGenerate, - }, - { - name: "validation runtime failure", - req: promptkit.RunRequest{ - PromptID: frameworkStructuredEventsPromptID, - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - }, - }, - client: &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `{"events":[]}`}}, - schemaDir: t.TempDir(), - want: promptkit.ErrValidation, - }, - } - - t.Setenv("PROMPTKIT_API_KEY", "test-secret") - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - schemaDir := tc.schemaDir - if schemaDir == "" { - schemaDir = frameworkSchemaDir - } - engine := newContractEngineWithOptions(t, schemaDir, promptkit.WithLLMClient(tc.client)) - _, err := engine.Run(context.Background(), tc.req) - if !errors.Is(err, tc.want) { - t.Fatalf("expected errors.Is(%v), got %v", tc.want, err) - } - if tc.notWant != nil && errors.Is(err, tc.notWant) { - t.Fatalf("did not expect errors.Is(%v), got %v", tc.notWant, err) - } - }) - } -} - -func TestSelectedProfileRawAPIKeyFailsConstruction(t *testing.T) { - profileDir := t.TempDir() - if err := os.WriteFile(filepath.Join(profileDir, "raw.yaml"), []byte(` -id: raw-profile -endpoint: http://localhost:8000/v1 -model: model -api_key: secret -`), 0644); err != nil { - t.Fatal(err) - } - - _, err := testfixture.NewEngine(testfixture.Config{ - PromptDir: frameworkPromptDir, - ProfileDir: profileDir, - SchemaDir: frameworkSchemaDir, - }) - if !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("expected ErrInvalidConfig, got %v", err) - } -} - -func TestSelectedProfileInvalidYAMLFailsConstruction(t *testing.T) { - profileDir := t.TempDir() - if err := os.WriteFile(filepath.Join(profileDir, "broken.yaml"), []byte(` -id: broken-profile -unknown_field: true -`), 0644); err != nil { - t.Fatal(err) - } - - _, err := testfixture.NewEngine(testfixture.Config{ - PromptDir: frameworkPromptDir, - ProfileDir: profileDir, - SchemaDir: frameworkSchemaDir, - }) - if !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("expected ErrInvalidConfig, got %v", err) - } -} - -func TestPromptRepositoryReadFailureFailsConstruction(t *testing.T) { - missingPromptDir := filepath.Join(t.TempDir(), "missing-prompts") - _, err := testfixture.NewEngine(testfixture.Config{ - PromptDir: missingPromptDir, - ProfileDir: frameworkProfileDir, - SchemaDir: frameworkSchemaDir, - }) - if !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("expected ErrInvalidConfig, got %v", err) - } -} - -func TestPromptDefinitionReadFailureFailsConstruction(t *testing.T) { - readErr := errors.New("definition read failed") - source := &publicDefinitionReadFailureFS{ - FS: fstest.MapFS{ - "prompts/target.yaml": &fstest.MapFile{Data: []byte("unread")}, - }, - target: "prompts/target.yaml", - err: readErr, - } - engine, err := testfixture.NewEngine(testfixture.Config{}, testfixture.WithPromptFS(source, "prompts")) - if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("NewEngine() = (%#v, %v), want nil with invalid-config identity", engine, err) - } -} - -func TestSelectedProfileRepositoryReadFailureFailsConstruction(t *testing.T) { - missingProfileDir := filepath.Join(t.TempDir(), "missing-profiles") - _, err := testfixture.NewEngine(testfixture.Config{ - PromptDir: frameworkPromptDir, - ProfileDir: missingProfileDir, - SchemaDir: frameworkSchemaDir, - }) - if !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("expected ErrInvalidConfig, got %v", err) - } -} - -func TestPrepareUsesBuiltInProfileWithoutProfileDir(t *testing.T) { - tests := []struct { - name string - profileID string - backendID string - }{ - { - name: "OpenRouter", - profileID: "mistral-small-3", - backendID: promptkit.BackendOpenRouter, - }, - { - name: "Rakestrawhome", - profileID: "rakestrawhome-gemma-4-31b", - backendID: promptkit.BackendRakestrawHome, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - engine, err := testfixture.NewEngine(testfixture.Config{ - PromptDir: frameworkPromptDir, - SchemaDir: frameworkSchemaDir, - }) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: tc.profileID, - APIKey: "test-key", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - }) - if err != nil { - t.Fatalf("expected built-in profile prepare to succeed, got %v", err) - } - if prepared.SelectedProfileID != tc.profileID || - prepared.SelectedBackendID != tc.backendID || - prepared.EffectiveModelParams.BackendID != tc.backendID { - t.Fatalf("unexpected built-in preparation: %#v", prepared) - } - }) - } -} - -func TestPromptDefaultProfileCanUseBuiltInProfile(t *testing.T) { - t.Setenv("OPENROUTER_API_KEY", "test-key") - promptDir := t.TempDir() - writePublicPromptFile(t, promptDir, "prompt.builtin.default", "mistral-small-3") - - engine, err := testfixture.NewEngine(testfixture.Config{PromptDir: promptDir}) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: "prompt.builtin.default", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - }, - }) - if err != nil { - t.Fatalf("expected built-in default profile prepare to succeed, got %v", err) - } - if prepared.SelectedProfileID != "mistral-small-3" { - t.Fatalf("unexpected selected profile: %q", prepared.SelectedProfileID) - } -} - -func TestCustomProfileOverridesBuiltInProfile(t *testing.T) { - t.Setenv("OPENROUTER_API_KEY", "test-key") - profileDir := t.TempDir() - writePublicProfileFile(t, profileDir, "mistral-small-3", "http://localhost:8000/v1", "custom-model") - - engine, err := testfixture.NewEngine(testfixture.Config{ - PromptDir: frameworkPromptDir, - ProfileDir: profileDir, - SchemaDir: frameworkSchemaDir, - }) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "mistral-small-3", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - }) - if err != nil { - t.Fatalf("expected custom profile prepare to succeed, got %v", err) - } - if prepared.EffectiveModelParams.Model != "custom-model" { - t.Fatalf("expected custom profile to override built-in, got %q", prepared.EffectiveModelParams.Model) - } -} - -func TestMalformedCustomProfileFailsConstruction(t *testing.T) { - t.Setenv("OPENROUTER_API_KEY", "test-key") - profileDir := t.TempDir() - if err := os.WriteFile(filepath.Join(profileDir, "mistral-small-3.yml"), []byte(` -id: mistral-small-3 -endpoint: http://localhost:8000/v1 -model: custom-model -unexpected: true -`), 0o644); err != nil { - t.Fatal(err) - } - - _, err := testfixture.NewEngine(testfixture.Config{ - PromptDir: frameworkPromptDir, - ProfileDir: profileDir, - SchemaDir: frameworkSchemaDir, - }) - if !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("expected invalid effective custom profile error, got %v", err) - } -} - -func TestPromptContentFilePathsAcrossSources(t *testing.T) { - type pathCase struct { - name string - directoryPath string - singleFilePath string - directoryTarget string - singleTarget string - absolute bool - symlink bool - wantErr string - } - tests := []pathCase{ - {name: "ordinary sibling", directoryPath: "sibling.tmpl", singleFilePath: "sibling.tmpl", directoryTarget: "nested/sibling.tmpl", singleTarget: "sibling.tmpl"}, - {name: "parent remains inside root", directoryPath: "../shared.tmpl", singleFilePath: "nested/../shared.tmpl", directoryTarget: "shared.tmpl", singleTarget: "shared.tmpl"}, - {name: "parent escapes root", directoryPath: "../../outside.tmpl", singleFilePath: "../outside.tmpl", wantErr: "escapes source root"}, - {name: "absolute path", absolute: true, wantErr: "must be relative"}, - {name: "leading whitespace preserved", directoryPath: " body.tmpl", singleFilePath: " body.tmpl", directoryTarget: "nested/ body.tmpl", singleTarget: " body.tmpl"}, - {name: "trailing whitespace preserved", directoryPath: "body.tmpl ", singleFilePath: "body.tmpl ", directoryTarget: "nested/body.tmpl ", singleTarget: "body.tmpl "}, - } - sources := []struct { - name string - singleFile bool - injectedFS bool - supportsSymlink bool - }{ - {name: "operating system directory", supportsSymlink: true}, - {name: "injected filesystem", injectedFS: true}, - {name: "single file", singleFile: true, supportsSymlink: true}, - } - - profile := testfixture.Profile{ID: "content-profile", Endpoint: "http://example.test/v1", Model: "content-model"} - const promptID = "content-path-prompt" - const body = "Exact content body." - - for _, source := range sources { - for _, tc := range tests { - t.Run(source.name+"/"+tc.name, func(t *testing.T) { - if tc.symlink && !source.supportsSymlink { - t.Skip("source does not expose operating-system symlink semantics") - } - - workspace := t.TempDir() - sourceRoot := filepath.Join(workspace, "prompts") - outsidePath := filepath.Join(workspace, "outside.tmpl") - if err := os.MkdirAll(filepath.Join(sourceRoot, "nested"), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(outsidePath, []byte("Outside root."), 0o644); err != nil { - t.Fatal(err) - } - - contentFile := tc.directoryPath - target := tc.directoryTarget - promptRelativePath := "nested/prompt.yaml" - if source.singleFile { - contentFile = tc.singleFilePath - target = tc.singleTarget - promptRelativePath = "prompt.yaml" - } - if tc.absolute { - contentFile = outsidePath - if source.injectedFS { - contentFile = "/outside.tmpl" - } - } - - promptDocument := []byte(fmt.Sprintf(` -id: %s -version: "1.0.0" -default_profile: content-profile -messages: - - role: user - content_file: %q -output: - format: text - validation_mode: none -`, promptID, contentFile)) - - var engine *promptkit.Engine - var err error - if source.injectedFS { - promptFS := fstest.MapFS{ - "prompts/nested/prompt.yaml": &fstest.MapFile{Data: promptDocument}, - "outside.tmpl": &fstest.MapFile{Data: []byte("Outside root.")}, - } - if target != "" { - promptFS["prompts/"+filepath.ToSlash(target)] = &fstest.MapFile{Data: []byte(body)} - } - engine, err = testfixture.NewEngine(testfixture.Config{}, testfixture.WithPromptFS(promptFS, "prompts"), testfixture.WithProfiles(profile)) - } else { - promptPath := filepath.Join(sourceRoot, filepath.FromSlash(promptRelativePath)) - if err := os.WriteFile(promptPath, promptDocument, 0o644); err != nil { - t.Fatal(err) - } - if target != "" { - targetPath := filepath.Join(sourceRoot, filepath.FromSlash(target)) - if err := os.MkdirAll(filepath.Dir(targetPath), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(targetPath, []byte(body), 0o644); err != nil { - t.Fatal(err) - } - } - if tc.symlink { - linkPath := filepath.Join(filepath.Dir(promptPath), "escape.tmpl") - if err := os.Symlink(outsidePath, linkPath); err != nil { - t.Skipf("symlinks are not supported: %v", err) - } - } - if source.singleFile { - engine, err = testfixture.NewEngine(testfixture.Config{}, testfixture.WithPromptFile(promptPath), testfixture.WithProfiles(profile)) - } else { - engine, err = testfixture.NewEngine(testfixture.Config{PromptDir: sourceRoot}, testfixture.WithProfiles(profile)) - } - } - if tc.wantErr != "" { - if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) || !strings.Contains(err.Error(), tc.wantErr) { - t.Fatalf("construction = (%#v, %v), want invalid configuration containing %q", engine, err, tc.wantErr) - } - return - } - if err != nil { - t.Fatalf("construct engine: %v", err) - } - - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: promptID}) - if err != nil { - t.Fatalf("prepare: %v", err) - } - if len(prepared.Messages) != 1 || prepared.Messages[0].Content != body { - t.Fatalf("expected exact content body, got %+v", prepared.Messages) - } - }) - } - } -} - -func TestPromptContentFileFailuresRejectConstruction(t *testing.T) { - promptFS := fstest.MapFS{ - "prompts/prompt.yaml": &fstest.MapFile{Data: []byte(` -id: rejected-content-path -version: "1.0.0" -messages: - - role: user - content_file: ../outside.tmpl -output: - format: text - validation_mode: none -`)}, - "outside.tmpl": &fstest.MapFile{Data: []byte("Outside root.")}, - } - engine, err := testfixture.NewEngine(testfixture.Config{}, testfixture.WithPromptFS(promptFS, "prompts")) - if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) || !strings.Contains(err.Error(), "content_file") { - t.Fatalf("construction = (%#v, %v), want content_file configuration error", engine, err) - } -} - -func TestPrepareWorksWithProfileFSOverBuiltIns(t *testing.T) { - profileFS := fstest.MapFS{ - "profiles/mistral-small-3.yaml": &fstest.MapFile{Data: []byte(` -id: mistral-small-3 -endpoint: http://profile-fs/v1 -model: profile-fs-model -`)}, - } - - engine, err := testfixture.NewEngine(testfixture.Config{ - PromptDir: frameworkPromptDir, - SchemaDir: frameworkSchemaDir, - }, testfixture.WithProfileFS(profileFS, "profiles")) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "mistral-small-3", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - }) - if err != nil { - t.Fatalf("expected prepare to succeed, got %v", err) - } - if prepared.EffectiveModelParams.Model != "profile-fs-model" { - t.Fatalf("expected profile fs to override built-in, got %q", prepared.EffectiveModelParams.Model) - } -} - -func TestPrepareWorksWithProfileFileOverBuiltIns(t *testing.T) { - profileDir := t.TempDir() - profilePath := filepath.Join(profileDir, "mistral-small-3.yaml") - if err := os.WriteFile(profilePath, []byte(` -id: mistral-small-3 -endpoint: http://profile-file/v1 -model: profile-file-model -`), 0o644); err != nil { - t.Fatal(err) - } - - engine, err := testfixture.NewEngine(testfixture.Config{ - PromptDir: frameworkPromptDir, - SchemaDir: frameworkSchemaDir, - }, testfixture.WithProfileFile(profilePath)) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "mistral-small-3", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - }) - if err != nil { - t.Fatalf("expected prepare to succeed, got %v", err) - } - if prepared.EffectiveModelParams.Model != "profile-file-model" { - t.Fatalf("expected profile file to override built-in, got %q", prepared.EffectiveModelParams.Model) - } -} - -func TestPrepareWorksWithInMemoryProfilesWithoutProfileFiles(t *testing.T) { - engine, err := testfixture.NewEngine(testfixture.Config{ - PromptDir: frameworkPromptDir, - SchemaDir: frameworkSchemaDir, - }, testfixture.WithProfiles(testfixture.Profile{ - ID: "memory-profile", - Endpoint: " https://memory-profile/nested/v1 ", - Model: "memory-model", - })) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "memory-profile", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - }) - if err != nil { - t.Fatalf("expected prepare to succeed, got %v", err) - } - if prepared.EffectiveModelParams.Model != "memory-model" { - t.Fatalf("expected in-memory profile model, got %q", prepared.EffectiveModelParams.Model) - } - if prepared.EffectiveModelParams.Endpoint != "https://memory-profile/nested/v1" { - t.Fatalf("expected normalized in-memory profile endpoint, got %q", prepared.EffectiveModelParams.Endpoint) - } -} - -func TestInMemoryProfilesOverrideBuiltInsAndProfileSources(t *testing.T) { - profileFS := fstest.MapFS{ - "profiles/mistral-small-3.yaml": &fstest.MapFile{Data: []byte(` -id: mistral-small-3 -endpoint: http://profile-fs/v1 -model: profile-fs-model -`)}, - } - - engine, err := testfixture.NewEngine(testfixture.Config{ - PromptDir: frameworkPromptDir, - SchemaDir: frameworkSchemaDir, - }, - testfixture.WithProfileFS(profileFS, "profiles"), - testfixture.WithProfiles(testfixture.Profile{ - ID: "mistral-small-3", - Endpoint: "http://memory-profile/v1", - Model: "memory-profile-model", - }), - ) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "mistral-small-3", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - }) - if err != nil { - t.Fatalf("expected prepare to succeed, got %v", err) - } - if prepared.EffectiveModelParams.Model != "memory-profile-model" { - t.Fatalf("expected in-memory profile to have highest precedence, got %q", prepared.EffectiveModelParams.Model) - } -} - -func TestWithProfilesRejectsDuplicateIDs(t *testing.T) { - _, err := testfixture.NewEngine(testfixture.Config{PromptDir: frameworkPromptDir}, - testfixture.WithProfiles( - testfixture.Profile{ID: "duplicate", Endpoint: "http://one/v1", Model: "one"}, - testfixture.Profile{ID: "duplicate", Endpoint: "http://two/v1", Model: "two"}, - ), - ) - if !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("expected ErrInvalidConfig, got %v", err) - } -} - -func TestProgrammaticProfilesAllowPartialDefinitions(t *testing.T) { - _, err := testfixture.NewEngine(testfixture.Config{PromptDir: frameworkPromptDir}, - testfixture.WithProfiles( - testfixture.Profile{ID: "base-profile", Endpoint: "https://base.example/v1"}, - testfixture.Profile{ID: "derived-profile", BaseProfileID: "base-profile"}, - ), - ) - if err != nil { - t.Fatalf("derived profile should be accepted during construction: %v", err) - } - - _, err = testfixture.NewEngine(testfixture.Config{PromptDir: frameworkPromptDir}, - testfixture.WithProfiles(testfixture.Profile{ID: "standalone-profile"}), - ) - if err != nil { - t.Fatalf("standalone partial profile should be accepted: %v", err) - } -} - -func TestWithProfilesRejectsInvalidExecutionSettings(t *testing.T) { - type testCase struct { - name string - profile testfixture.Profile - } - tests := []testCase{ - {name: "non-finite temperature", profile: testfixture.Profile{Temperature: math.NaN()}}, - {name: "non-finite top p", profile: testfixture.Profile{TopP: math.Inf(-1)}}, - } - if strconv.IntSize == 64 { - durationLimit := int64(math.MaxInt64 / int64(time.Second)) - tests = append(tests, testCase{name: "unrepresentable timeout", profile: testfixture.Profile{TimeoutSeconds: int(durationLimit) + 1}}) - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - tt.profile.ID = "invalid-settings" - tt.profile.Endpoint = "http://example.test/v1" - tt.profile.Model = "model" - _, err := testfixture.NewEngine(testfixture.Config{PromptDir: frameworkPromptDir}, - testfixture.WithProfiles(tt.profile), - ) - if !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("expected ErrInvalidConfig, got %v", err) - } - }) - } -} - -func TestEndpointValidationMapsPublicErrorCategories(t *testing.T) { - t.Run("backend registration is invalid configuration", func(t *testing.T) { - _, err := testfixture.NewEngine( - testfixture.Config{PromptDir: frameworkPromptDir}, - testfixture.WithBackend(testfixture.Backend{ID: "invalid", Endpoint: "/v1"}), - ) - if !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("expected ErrInvalidConfig, got %v", err) - } - }) - - t.Run("in-memory profile is invalid configuration", func(t *testing.T) { - _, err := testfixture.NewEngine( - testfixture.Config{PromptDir: frameworkPromptDir}, - testfixture.WithProfiles(testfixture.Profile{ - ID: "invalid", - Endpoint: "https://provider.example/v1?mode=chat", - Model: "model", - }), - ) - if !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("expected ErrInvalidConfig, got %v", err) - } - }) - - t.Run("file profile is a profile-load failure", func(t *testing.T) { - profileFS := fstest.MapFS{ - "profiles/invalid.yaml": &fstest.MapFile{Data: []byte(` -id: invalid -endpoint: 'https://provider.example/v1#chat' -model: model -`)}, - } - engine, err := testfixture.NewEngine( - testfixture.Config{PromptDir: frameworkPromptDir}, - testfixture.WithProfileFS(profileFS, "profiles"), - ) - if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("expected invalid configuration at construction, got (%#v, %v)", engine, err) - } - }) - - t.Run("request override is an invalid request", func(t *testing.T) { - engine, err := testfixture.NewEngine( - testfixture.Config{PromptDir: frameworkPromptDir}, - testfixture.WithProfiles(testfixture.Profile{ - ID: "valid", - Endpoint: "https://provider.example/v1", - Model: "model", - }), - ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - _, err = engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "valid", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - Execution: &promptkit.ExecutionTargetOverride{Endpoint: "ftp://provider.example/v1"}, - }) - if !errors.Is(err, promptkit.ErrInvalidRequest) { - t.Fatalf("expected ErrInvalidRequest, got %v", err) - } - }) -} - -func TestOpenAICompatibleProfileRunsThroughNormalProfilePath(t *testing.T) { - fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}} - prof := testfixture.OpenAICompatibleProfile(testfixture.OpenAICompatibleProfileConfig{ - ID: "template-profile", - BackendID: "openrouter", - Endpoint: "http://template/v1", - Model: "template-model", - APIKeyRequired: true, - ExtraParams: map[string]any{ - "provider": "template", - }, - }) - - engine, err := testfixture.NewEngine(testfixture.Config{ - PromptDir: frameworkPromptDir, - SchemaDir: frameworkSchemaDir, - }, testfixture.WithProfiles(prof), promptkit.WithLLMClient(fake)) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - _, err = engine.Run(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "template-profile", - APIKey: "template-key", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - }) - if err != nil { - t.Fatalf("expected run to succeed, got %v", err) - } - if len(fake.requests) != 1 { - t.Fatalf("expected one request, got %d", len(fake.requests)) - } - if fake.requests[0].Target.BackendID != promptkit.BackendOpenRouter || - fake.requests[0].Target.Model != "template-model" || - fake.requests[0].APIKey != "template-key" { - t.Fatalf("unexpected generated request: %+v", fake.requests[0]) - } - if !reflect.DeepEqual(fake.requests[0].Target.ExtraParams, map[string]any{"provider": "template"}) { - t.Fatalf("unexpected extra params: %#v", fake.requests[0].Target.ExtraParams) - } -} - -func TestOpenAICompatibleProfileMapsEveryField(t *testing.T) { - extraParams := map[string]any{"provider_option": "distinct-extra-params"} - got := testfixture.OpenAICompatibleProfile(testfixture.OpenAICompatibleProfileConfig{ - ID: "distinct-id", - BaseProfileID: "distinct-base", - BackendID: "distinct-backend", - Endpoint: "https://distinct.example/v1", - Model: "distinct-model", - APIKeyRequired: true, - Temperature: 0.25, - MaxTokens: 321, - TopP: 0.75, - TimeoutSeconds: 43, - ServiceTier: "distinct-service-tier", - ReasoningEffort: "distinct-reasoning-effort", - ExtraParams: extraParams, - }) - want := testfixture.Profile{ - ID: "distinct-id", - BaseProfileID: "distinct-base", - BackendID: "distinct-backend", - Endpoint: "https://distinct.example/v1", - Model: "distinct-model", - Temperature: 0.25, - MaxTokens: 321, - TopP: 0.75, - TimeoutSeconds: 43, - ServiceTier: "distinct-service-tier", - ReasoningEffort: "distinct-reasoning-effort", - APIKeyRequired: true, - ExtraParams: map[string]any{"provider_option": "distinct-extra-params"}, - } - if !reflect.DeepEqual(got, want) { - t.Fatalf("profile mapping:\ngot %#v\nwant %#v", got, want) - } - - extraParams["added_after_construction"] = true - if _, ok := got.ExtraParams["added_after_construction"]; ok { - t.Fatal("profile retained the configuration map") - } -} - -func TestEngineRunLayersTransportAndGenerationTimeouts(t *testing.T) { - intPointer := func(value int) *int { - return &value - } - - tests := []struct { - name string - configTimeout time.Duration - suppliedClientTimeout time.Duration - profileTimeoutSeconds int - requestTimeoutSeconds *int - callerTimeout time.Duration - wantRemainingAtRequest time.Duration - }{ - { - name: "positive supplied client cap takes precedence over config", - configTimeout: 2 * time.Second, - suppliedClientTimeout: 6 * time.Second, - wantRemainingAtRequest: 6 * time.Second, - }, - { - name: "zero supplied client timeout inherits config cap", - configTimeout: 5 * time.Second, - wantRemainingAtRequest: 5 * time.Second, - }, - { - name: "zero configuration uses ten minute transport default", - wantRemainingAtRequest: 10 * time.Minute, - }, - { - name: "negative configuration uses ten minute transport default", - configTimeout: -2 * time.Second, - suppliedClientTimeout: -3 * time.Second, - wantRemainingAtRequest: 10 * time.Minute, - }, - { - name: "profile deadline is shorter than transport cap", - suppliedClientTimeout: 6 * time.Second, - profileTimeoutSeconds: 4, - wantRemainingAtRequest: 4 * time.Second, - }, - { - name: "request deadline is shorter than profile and transport limits", - suppliedClientTimeout: 6 * time.Second, - profileTimeoutSeconds: 4, - requestTimeoutSeconds: intPointer(2), - wantRemainingAtRequest: 2 * time.Second, - }, - { - name: "explicit zero removes generation deadline but retains transport cap", - suppliedClientTimeout: 5 * time.Second, - profileTimeoutSeconds: 2, - requestTimeoutSeconds: intPointer(0), - wantRemainingAtRequest: 5 * time.Second, - }, - { - name: "framework default remains layered with shorter transport cap", - configTimeout: 7 * time.Second, - suppliedClientTimeout: 3 * time.Second, - wantRemainingAtRequest: 3 * time.Second, - }, - { - name: "caller deadline remains layered with other limits", - suppliedClientTimeout: 6 * time.Second, - profileTimeoutSeconds: 4, - callerTimeout: 2 * time.Second, - wantRemainingAtRequest: 2 * time.Second, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - var ( - sawDeadline bool - remaining time.Duration - ) - transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { - deadline, ok := req.Context().Deadline() - sawDeadline = ok - if ok { - remaining = time.Until(deadline) - } - return &http.Response{ - StatusCode: http.StatusOK, - Status: "200 OK", - Header: make(http.Header), - Body: io.NopCloser(strings.NewReader( - `{"choices":[{"message":{"content":"ok"}}]}`, - )), - Request: req, - }, nil - }) - httpClient := &http.Client{ - Timeout: tc.suppliedClientTimeout, - Transport: transport, - } - engine, err := testfixture.NewEngine(testfixture.Config{ - PromptDir: frameworkPromptDir, - SchemaDir: frameworkSchemaDir, - Timeout: tc.configTimeout, - HTTPClient: httpClient, - }, testfixture.WithProfiles(testfixture.Profile{ - ID: "layered-timeout", - Endpoint: "http://timeout.test/v1", - Model: "timeout-model", - TimeoutSeconds: tc.profileTimeoutSeconds, - })) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - ctx := context.Background() - cancel := func() {} - if tc.callerTimeout > 0 { - ctx, cancel = context.WithTimeout(ctx, tc.callerTimeout) - } - defer cancel() - - _, err = engine.Run(ctx, promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "layered-timeout", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - Execution: &promptkit.ExecutionTargetOverride{ - TimeoutSeconds: tc.requestTimeoutSeconds, - }, - }) - if err != nil { - t.Fatalf("expected run to succeed, got %v", err) - } - if !sawDeadline { - t.Fatal("expected outbound request context to have a deadline") - } - - const deadlineTolerance = 750 * time.Millisecond - if remaining < tc.wantRemainingAtRequest-deadlineTolerance || - remaining > tc.wantRemainingAtRequest+50*time.Millisecond { - t.Fatalf( - "unexpected request deadline: remaining=%v want approximately %v", - remaining, - tc.wantRemainingAtRequest, - ) - } - }) - } -} - -func TestOpenAICompatibleProfileDefersExtraParamsValidation(t *testing.T) { - cyclic := map[string]any{} - cyclic["self"] = cyclic - - prof := testfixture.OpenAICompatibleProfile(testfixture.OpenAICompatibleProfileConfig{ - ID: "cyclic-template-profile", - Endpoint: "http://cyclic-template/v1", - Model: "cyclic-template-model", - ExtraParams: cyclic, - }) - - _, err := testfixture.NewEngine(testfixture.Config{PromptDir: frameworkPromptDir}, - testfixture.WithProfiles(prof), - ) - if !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("expected ErrInvalidConfig, got %v", err) - } -} - -func TestOpenAICompatibleProfileNestedExtraParamsRunThroughWithProfiles(t *testing.T) { - fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}} - nested := map[string]any{ - "labels": map[string]string{"route": "primary"}, - "ids": []int{1, 2, 3}, - } - extraParams := map[string]any{ - "nested": nested, - } - prof := testfixture.OpenAICompatibleProfile(testfixture.OpenAICompatibleProfileConfig{ - ID: "nested-template-profile", - Endpoint: "http://nested-template/v1", - Model: "nested-template-model", - ExtraParams: extraParams, - }) - extraParams["added"] = "mutated-after-construction" - - engine, err := testfixture.NewEngine(testfixture.Config{ - PromptDir: frameworkPromptDir, - SchemaDir: frameworkSchemaDir, - }, testfixture.WithProfiles(prof), promptkit.WithLLMClient(fake)) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - nested["added"] = "mutated-after-construction" - - _, err = engine.Run(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "nested-template-profile", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - }) - if err != nil { - t.Fatalf("expected run to succeed, got %v", err) - } - want := map[string]any{ - "nested": map[string]any{ - "labels": map[string]any{"route": "primary"}, - "ids": []any{int64(1), int64(2), int64(3)}, - }, - } - if !reflect.DeepEqual(fake.requests[0].Target.ExtraParams, want) { - t.Fatalf("unexpected extra params:\ngot=%#v\nwant=%#v", fake.requests[0].Target.ExtraParams, want) - } -} - -func TestInMemoryProfileAPIKeyRequiredBehavior(t *testing.T) { - engine, err := testfixture.NewEngine(testfixture.Config{ - PromptDir: frameworkPromptDir, - SchemaDir: frameworkSchemaDir, - }, testfixture.WithProfiles(testfixture.Profile{ - ID: "requires-key", - Endpoint: "http://requires-key/v1", - Model: "requires-key-model", - APIKeyRequired: true, - })) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - req := promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "requires-key", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - } - _, err = engine.Prepare(context.Background(), req) - if !errors.Is(err, promptkit.ErrInvalidRequest) { - t.Fatalf("expected ErrInvalidRequest without API key, got %v", err) - } - req.APIKey = "direct-required-key" - if _, err := engine.Prepare(context.Background(), req); err != nil { - t.Fatalf("expected direct API key to satisfy APIKeyRequired, got %v", err) - } -} - -func TestInMemoryProfileWithoutAPIKeyRequiredWorksWithoutKey(t *testing.T) { - engine, err := testfixture.NewEngine(testfixture.Config{ - PromptDir: frameworkPromptDir, - SchemaDir: frameworkSchemaDir, - }, testfixture.WithProfiles(testfixture.Profile{ - ID: "no-key-required", - Endpoint: "http://no-key/v1", - Model: "no-key-model", - })) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - _, err = engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "no-key-required", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - }) - if err != nil { - t.Fatalf("expected prepare without API key to succeed, got %v", err) - } -} - -func TestInMemoryProfileExtraParamsAreCopiedAcrossPublicBoundary(t *testing.T) { - fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}} - labels := map[string]string{"route": "primary"} - ids := []int{1, 2, 3} - extraParams := map[string]any{ - "labels": labels, - "ids": ids, - } - - engine, err := testfixture.NewEngine(testfixture.Config{ - PromptDir: frameworkPromptDir, - SchemaDir: frameworkSchemaDir, - }, - testfixture.WithProfiles(testfixture.Profile{ - ID: "copy-profile", - Endpoint: "http://copy/v1", - Model: "copy-model", - ExtraParams: extraParams, - }), - promptkit.WithLLMClient(fake), - ) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - labels["route"] = "mutated-before-run" - ids[0] = 99 - extraParams["added"] = "mutated" - - _, err = engine.Run(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "copy-profile", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - }) - if err != nil { - t.Fatalf("expected run to succeed, got %v", err) - } - want := map[string]any{ - "labels": map[string]any{"route": "primary"}, - "ids": []any{int64(1), int64(2), int64(3)}, - } - if !reflect.DeepEqual(fake.requests[0].Target.ExtraParams, want) { - t.Fatalf("captured extra params changed after mutation:\ngot=%#v\nwant=%#v", fake.requests[0].Target.ExtraParams, want) - } -} - -func TestWithProfilesCopiesDefinitionsWhenApplied(t *testing.T) { - nested := map[string]any{"value": "when option is created"} - option := testfixture.WithProfiles(testfixture.Profile{ - ID: "profile", - Endpoint: "http://copy.example/v1", - Model: "copy-model", - ExtraParams: map[string]any{"nested": nested}, - }) - nested["value"] = "when option is applied" - - engine, err := testfixture.NewEngine(testfixture.Config{PromptDir: frameworkPromptDir}, option) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - nested["value"] = "after construction" - - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "profile", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - }) - if err != nil { - t.Fatalf("prepare with copied profile: %v", err) - } - if got := prepared.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["value"]; got != "when option is applied" { - t.Fatalf("profile copy value = %q, want value at option application", got) - } -} - -func TestWithProfilesRejectsInvalidExtraParams(t *testing.T) { - tests := []struct { - name string - extraParams map[string]any - }{ - {name: "function", extraParams: map[string]any{"bad": func() {}}}, - {name: "channel", extraParams: map[string]any{"bad": make(chan struct{})}}, - {name: "struct", extraParams: map[string]any{"bad": struct{ Name string }{Name: "bad"}}}, - {name: "non string map key", extraParams: map[string]any{"bad": map[int]string{1: "one"}}}, - {name: "nan", extraParams: map[string]any{"bad": math.NaN()}}, - {name: "positive infinity", extraParams: map[string]any{"bad": math.Inf(1)}}, - {name: "negative infinity", extraParams: map[string]any{"bad": math.Inf(-1)}}, - {name: "excessively large value", extraParams: map[string]any{"bad": excessivelyLargeJSONValue()}}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - _, err := testfixture.NewEngine(testfixture.Config{PromptDir: frameworkPromptDir}, - testfixture.WithProfiles(testfixture.Profile{ - ID: "invalid-extra-params", - Endpoint: "http://invalid/v1", - Model: "invalid-model", - ExtraParams: tc.extraParams, - }), - ) - if !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("expected ErrInvalidConfig, got %v", err) - } - }) - } -} - -func TestWithProfilesRejectsCyclicExtraParams(t *testing.T) { - cyclicMap := map[string]any{} - cyclicMap["self"] = cyclicMap - cyclicSlice := []any{nil} - cyclicSlice[0] = cyclicSlice - - tests := []struct { - name string - extraParams map[string]any - }{ - {name: "map", extraParams: cyclicMap}, - {name: "slice", extraParams: map[string]any{"cycle": cyclicSlice}}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - _, err := testfixture.NewEngine(testfixture.Config{PromptDir: frameworkPromptDir}, - testfixture.WithProfiles(testfixture.Profile{ - ID: "cyclic-extra-params", - Endpoint: "http://cyclic/v1", - Model: "cyclic-model", - ExtraParams: tc.extraParams, - }), - ) - if !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("expected ErrInvalidConfig, got %v", err) - } - }) - } -} - -func TestPublicSchemaGraphErrorsHaveSourceParity(t *testing.T) { - tests := []struct { - name string - files map[string]string - valid bool - }{ - { - name: "invalid keyword", - files: map[string]string{"root.json": `{"type":42}`}, - }, - { - name: "malformed direct reference", - files: map[string]string{ - "root.json": `{"$ref":"child.json"}`, - "child.json": `{`, - }, - }, - { - name: "missing direct reference", - files: map[string]string{"root.json": `{"$ref":"missing.json"}`}, - }, - { - name: "malformed second-level reference", - files: map[string]string{ - "root.json": `{"$ref":"child.json"}`, - "child.json": `{"$ref":"grandchild.json"}`, - "grandchild.json": `{`, - }, - }, - { - name: "missing second-level reference", - files: map[string]string{ - "root.json": `{"$ref":"child.json"}`, - "child.json": `{"$ref":"missing.json"}`, - }, - }, - { - name: "unsupported referenced dialect", - files: map[string]string{ - "root.json": `{"$ref":"child.json"}`, - "child.json": `{"$schema":"http://json-schema.org/draft-07/schema#","type":"integer"}`, - }, - }, - { - name: "escaping reference", - files: map[string]string{ - "root.json": `{"$ref":"../outside.json"}`, - "../outside.json": `{"type":"integer"}`, - }, - }, - { - name: "remote reference", - files: map[string]string{"root.json": `{"$ref":"https://example.test/schema.json"}`}, - }, - { - name: "valid multi-document graph", - files: map[string]string{ - "root.json": `{ - "type":"object", - "required":["value"], - "properties":{"value":{"$ref":"child.json"}} -}`, - "child.json": `{"$ref":"nested/value.json"}`, - "nested/value.json": `{"type":"integer","minimum":2}`, - }, - valid: true, - }, - } - - for _, source := range []string{"operating system files", "fs.FS"} { - t.Run(source, func(t *testing.T) { - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `{"value":3}`}} - engine := newPublicSchemaGraphEngine(t, source, tc.files, client) - result, err := engine.Run(context.Background(), promptkit.RunRequest{ - PromptID: "schema.graph.prompt", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - }, - }) - if tc.valid { - if err != nil { - t.Fatalf("run valid schema graph: %v", err) - } - if result.Validation.Status != promptkit.ValidationPassed || !result.Validation.IsValid { - t.Fatalf("validation = %+v, want passed", result.Validation) - } - if len(client.requests) != 1 { - t.Fatalf("generation calls = %d, want 1", len(client.requests)) - } - return - } - if !errors.Is(err, promptkit.ErrValidation) { - t.Fatalf("error = %v, want ErrValidation", err) - } - if result != nil { - t.Fatalf("result = %+v, want nil", result) - } - if len(client.requests) != 0 { - t.Fatalf("invalid schema reached generation: %+v", client.requests) - } - }) - } - }) - } -} - -func TestRunReadsSchemaGraphOncePerOperation(t *testing.T) { - source := &countingSchemaFS{ - FS: fstest.MapFS{ - "schemas/root.json": &fstest.MapFile{Data: []byte(`{ - "type":"object", - "required":["value"], - "properties":{"value":{"$ref":"child.json"}} -}`)}, - "schemas/child.json": &fstest.MapFile{Data: []byte(`{"$ref":"nested/value.json"}`)}, - "schemas/nested/value.json": &fstest.MapFile{Data: []byte(`{"type":"integer","minimum":2}`)}, - }, - reads: make(map[string]int), - } - client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `{"value":3}`}} - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(publicStructuredPromptFS("schema.count.prompt", "root.json"), "prompts"), - promptkit.WithSchemaFS(source, "schemas"), - testfixture.WithProfiles(testfixture.Profile{ - ID: "contract-fast", Endpoint: "http://example.test/v1", Model: "schema-model", - }), - promptkit.WithLLMClient(client), - ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - request := promptkit.RunRequest{ - PromptID: "schema.count.prompt", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - }, - } - - for operation := 1; operation <= 2; operation++ { - result, err := engine.Run(context.Background(), request) - if err != nil { - t.Fatalf("run operation %d: %v", operation, err) - } - if !result.Validation.IsValid { - t.Fatalf("run operation %d validation = %+v", operation, result.Validation) - } - for _, name := range []string{"schemas/root.json", "schemas/child.json", "schemas/nested/value.json"} { - if got := source.readCount(name); got != operation { - t.Fatalf("after operation %d, reads for %q = %d, want %d", operation, name, got, operation) - } - } - } -} - -func TestPreparePreservesValidationCancellationIdentity(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - source := &cancelingPublicSchemaFS{ - FS: fstest.MapFS{ - "schemas/schema.json": &fstest.MapFile{Data: []byte(`{"type":"object"}`)}, - }, - target: "schemas/schema.json", - cancel: cancel, - } - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(publicStructuredPromptFS("schema.cancel.prompt", "schema.json"), "prompts"), - promptkit.WithSchemaFS(source, "schemas"), - testfixture.WithProfiles(testfixture.Profile{ - ID: "contract-fast", Endpoint: "http://example.test/v1", Model: "schema-model", - }), - ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - - prepared, err := engine.Prepare(ctx, promptkit.RunRequest{ - PromptID: "schema.cancel.prompt", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - }, - }) - if prepared != nil || !errors.Is(err, promptkit.ErrValidation) || !errors.Is(err, context.Canceled) { - t.Fatalf("Prepare() = (%v, %v), want nil result with validation and context identities", prepared, err) - } - if reads := source.readCount(); reads != 1 { - t.Fatalf("schema reads = %d, want 1", reads) - } -} - -func TestPrepareRejectsExcessivelyDeepStructuredOutputSchema(t *testing.T) { - schema, err := json.Marshal(excessivelyDeepStructuredOutputSchema()) - if err != nil { - t.Fatalf("marshal schema: %v", err) - } - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(publicStructuredPromptFS("schema.deep.prompt", "schema.json"), "prompts"), - promptkit.WithSchemaFS(fstest.MapFS{ - "schemas/schema.json": &fstest.MapFile{Data: schema}, - }, "schemas"), - testfixture.WithProfiles(testfixture.Profile{ - ID: "contract-fast", Endpoint: "http://example.test/v1", Model: "schema-model", - }), - ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: "schema.deep.prompt", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - }, - }) - if prepared != nil || !errors.Is(err, promptkit.ErrValidation) { - t.Fatalf("Prepare() = (%#v, %v), want nil with ErrValidation", prepared, err) - } -} - -func TestPreparedStructuredOutputRetainsExactSchemaNumbers(t *testing.T) { - const schema = `{ - "type": "number", - "const": 9007199254740993, - "minimum": 0.123456789012345678901234567890, - "maximum": 1e400, - "multipleOf": 0.0000000000000000001 -}` - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(publicStructuredPromptFS("exact.schema.prompt", "events.schema.json"), "prompts"), - promptkit.WithSchemaFS(fstest.MapFS{ - "schemas/events.schema.json": &fstest.MapFile{Data: []byte(schema)}, - }, "schemas"), - testfixture.WithProfiles(testfixture.Profile{ - ID: "contract-fast", Endpoint: "http://example.test/v1", Model: "exact-model", - }), - ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: "exact.schema.prompt", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - }, - }) - if err != nil { - t.Fatalf("prepare structured prompt: %v", err) - } - if prepared.StructuredOutput == nil || prepared.StructuredOutput.JSONSchema == nil { - t.Fatalf("structured output = %#v, want JSON Schema", prepared.StructuredOutput) - } - document, ok := prepared.StructuredOutput.JSONSchema.Schema.(map[string]any) - if !ok { - t.Fatalf("public schema = %#v, want object", prepared.StructuredOutput.JSONSchema.Schema) - } - want := map[string]string{ - "const": "9007199254740993", - "minimum": "0.123456789012345678901234567890", - "maximum": "1e400", - "multipleOf": "0.0000000000000000001", - } - for name, wantNumber := range want { - got, ok := document[name].(json.Number) - if !ok { - t.Fatalf("public schema field %q = %#v, want json.Number", name, document[name]) - } - if got.String() != wantNumber { - t.Fatalf("public schema field %q = %q, want %q", name, got, wantNumber) - } - } -} - -func TestRunStructuredOutputWorksWithSchemaFS(t *testing.T) { - fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `{"events":[]}`}} - engine, err := testfixture.NewEngine(testfixture.Config{ - PromptDir: t.TempDir(), - ProfileDir: frameworkProfileDir, - SchemaDir: t.TempDir(), - }, - testfixture.WithPromptFS(publicStructuredPromptFS("schema.fs.prompt", "events.schema.json"), "prompts"), - promptkit.WithSchemaFS(publicSchemaFS(), "schemas"), - promptkit.WithLLMClient(fake), - ) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - result, err := engine.Run(context.Background(), promptkit.RunRequest{ - PromptID: "schema.fs.prompt", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - }, - }) - if err != nil { - t.Fatalf("expected run to succeed, got %v", err) - } - if result.Validation.Status != promptkit.ValidationPassed || !result.Validation.IsValid { - t.Fatalf("expected schema validation to pass, got %+v", result.Validation) - } - if len(fake.requests) != 1 || fake.requests[0].StructuredOutput == nil { - t.Fatalf("expected structured output request, got %+v", fake.requests) - } -} - -func TestRunStructuredOutputWorksWithSchemaFile(t *testing.T) { - schemaDir := t.TempDir() - schemaPath := filepath.Join(schemaDir, "events.schema.json") - if err := os.WriteFile(schemaPath, []byte(publicSchemaJSON()), 0o644); err != nil { - t.Fatal(err) - } - - fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: `{"events":[]}`}} - engine, err := testfixture.NewEngine(testfixture.Config{ - ProfileDir: frameworkProfileDir, - }, - testfixture.WithPromptFS(publicStructuredPromptFS("schema.file.prompt", "events.schema.json"), "prompts"), - promptkit.WithSchemaFile(schemaPath), - promptkit.WithLLMClient(fake), - ) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - - result, err := engine.Run(context.Background(), promptkit.RunRequest{ - PromptID: "schema.file.prompt", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - }, - }) - if err != nil { - t.Fatalf("expected run to succeed, got %v", err) - } - if result.Validation.Status != promptkit.ValidationPassed || !result.Validation.IsValid { - t.Fatalf("expected schema validation to pass, got %+v", result.Validation) - } -} - -func TestSingleFileSourcesPreserveLeadingWhitespaceInPaths(t *testing.T) { - profile := testfixture.Profile{ - ID: "exact-path-profile", - Endpoint: "http://example.test/v1", - Model: "exact-path-model", - } - promptDocument := []byte(` -id: exact-path-prompt -version: "1.0.0" -default_profile: exact-path-profile -messages: - - role: user - content: exact path -output: - format: text - validation_mode: none -`) - profileDocument := []byte(` -id: exact-path-profile -endpoint: http://example.test/v1 -model: exact-path-model -`) - - tests := []struct { - name string - baseName string - content []byte - engine func(string) (*promptkit.Engine, error) - }{ - { - name: "prompt", - baseName: "prompt.yaml", - content: promptDocument, - engine: func(path string) (*promptkit.Engine, error) { - return testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFile(path), - testfixture.WithProfiles(profile), - ) - }, - }, - { - name: "profile", - baseName: "profile.yaml", - content: profileDocument, - engine: func(path string) (*promptkit.Engine, error) { - return testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("exact-path-prompt", "exact-path-profile", "exact path"), "."), - testfixture.WithProfileFile(path), - ) - }, - }, - { - name: "schema", - baseName: "schema.json", - content: []byte(publicSchemaJSON()), - engine: func(path string) (*promptkit.Engine, error) { - promptSource := fstest.MapFS{ - "prompt.yaml": &fstest.MapFile{Data: []byte(fmt.Sprintf(` -id: exact-path-prompt -version: "1.0.0" -default_profile: exact-path-profile -messages: - - role: user - content: exact path -output: - format: json - validation_mode: json_schema - schema_path: %q -`, filepath.Base(path)))}, - } - return testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(promptSource, "."), - testfixture.WithProfiles(profile), - promptkit.WithSchemaFile(path), - ) - }, - }, - } - positions := []struct { - name string - prefix string - suffix string - }{ - {name: "leading whitespace", prefix: " "}, - } - - for _, tc := range tests { - for _, position := range positions { - t.Run(tc.name+"/"+position.name, func(t *testing.T) { - path := filepath.Join(t.TempDir(), position.prefix+tc.baseName+position.suffix) - if err := os.WriteFile(path, tc.content, 0o644); err != nil { - t.Fatalf("write source file: %v", err) - } - engine, err := tc.engine(path) - if err != nil { - t.Fatalf("construct engine from exact path %q: %v", path, err) - } - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "exact-path-prompt"}) - if err != nil { - t.Fatalf("prepare from exact path %q: %v", path, err) - } - if prepared.EffectiveModelParams.Model != "exact-path-model" { - t.Fatalf("prepared model = %q, want exact-path-model", prepared.EffectiveModelParams.Model) - } - }) - } - } -} - -func TestSourceOptionsRejectInvalidInputs(t *testing.T) { - missingFile := filepath.Join(t.TempDir(), "missing.yaml") - directoryPath := t.TempDir() - - tests := []struct { - name string - opt any - }{ - {name: "prompt fs nil", opt: testfixture.WithPromptFS(nil, "prompts")}, - {name: "prompt fs empty root", opt: testfixture.WithPromptFS(fstest.MapFS{}, "")}, - {name: "prompt file empty", opt: testfixture.WithPromptFile("")}, - {name: "prompt file missing", opt: testfixture.WithPromptFile(missingFile)}, - {name: "prompt file directory", opt: testfixture.WithPromptFile(directoryPath)}, - {name: "profile fs nil", opt: testfixture.WithProfileFS(nil, "profiles")}, - {name: "profile fs empty root", opt: testfixture.WithProfileFS(fstest.MapFS{}, "")}, - {name: "profile file empty", opt: testfixture.WithProfileFile("")}, - {name: "fallback profile fs nil", opt: testfixture.WithFallbackProfileFS(nil, "profiles")}, - {name: "fallback profile fs empty root", opt: testfixture.WithFallbackProfileFS(fstest.MapFS{}, "")}, - {name: "schema fs nil", opt: promptkit.WithSchemaFS(nil, "schemas")}, - {name: "schema fs empty root", opt: promptkit.WithSchemaFS(fstest.MapFS{}, "")}, - {name: "schema file empty", opt: promptkit.WithSchemaFile("")}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - _, err := testfixture.NewEngine(testfixture.Config{PromptDir: frameworkPromptDir}, tc.opt) - if !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("expected ErrInvalidConfig, got %v", err) - } - }) - } - - t.Run("later valid replacement does not hide invalid option", func(t *testing.T) { - _, err := testfixture.NewEngine(testfixture.Config{PromptDir: frameworkPromptDir}, - testfixture.WithProfileFS(nil, "profiles"), - testfixture.WithProfileFS(contractProfileFS("profile", "model"), "."), - ) - if !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("expected ErrInvalidConfig, got %v", err) - } - }) -} - -func TestPackageOptionsComposeFromSlice(t *testing.T) { - fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}} - options := []any{ - nil, - testfixture.WithProfiles(testfixture.Profile{ - ID: "slice-profile", - Endpoint: "http://slice/v1", - Model: "slice-model", - }), - promptkit.WithLLMClient(fake), - } - - engine, err := testfixture.NewEngine(testfixture.Config{ - PromptDir: frameworkPromptDir, - SchemaDir: frameworkSchemaDir, - }, options...) - if err != nil { - t.Fatalf("expected package-provided options to compose, got %v", err) - } - - _, err = engine.Run(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "slice-profile", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - }) - if err != nil { - t.Fatalf("expected run with composed options to succeed, got %v", err) - } - if len(fake.requests) != 1 { - t.Fatalf("expected one generate request, got %d", len(fake.requests)) - } - if fake.requests[0].Target.Model != "slice-model" { - t.Fatalf("expected profile from composed options, got %q", fake.requests[0].Target.Model) - } -} - -func TestExtraParamsTypedNestedValuesAreCopiedAcrossPublicBoundary(t *testing.T) { - fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}} - engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(fake)) - - labels := map[string]string{"route": "primary"} - counts := map[string]int{"retry_budget": 2} - weights := []float64{0.25, 0.75} - ids := []int{1, 2, 3} - nested := map[string]any{ - "labels": labels, - "counts": counts, - "weights": weights, - "ids": ids, - } - extraParams := map[string]any{ - "labels": labels, - "counts": counts, - "nested": nested, - } - - _, err := engine.Run(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - Execution: &promptkit.ExecutionTargetOverride{ExtraParams: extraParams}, - }) - if err != nil { - t.Fatalf("expected run to succeed, got %v", err) - } - if len(fake.requests) != 1 { - t.Fatalf("expected one generate request, got %d", len(fake.requests)) - } - - captured := fake.requests[0].Target.ExtraParams - labels["route"] = "mutated" - counts["retry_budget"] = 99 - weights[0] = 9.9 - ids[0] = 99 - nested["added"] = "mutated" - extraParams["new_top_level"] = "mutated" - - want := map[string]any{ - "labels": map[string]string{"route": "primary"}, - "counts": map[string]int{"retry_budget": 2}, - "nested": map[string]any{ - "labels": map[string]string{"route": "primary"}, - "counts": map[string]int{"retry_budget": 2}, - "weights": []float64{0.25, 0.75}, - "ids": []int{1, 2, 3}, - }, - } - if !reflect.DeepEqual(captured, want) { - t.Fatalf("captured extra_params changed after mutating source:\ngot=%#v\nwant=%#v", captured, want) - } -} - -func TestRunRejectsInvalidExtraParams(t *testing.T) { - cyclicMap := map[string]any{} - cyclicMap["self"] = cyclicMap - cyclicSlice := []any{nil} - cyclicSlice[0] = cyclicSlice - - tests := []struct { - name string - extraParams map[string]any - }{ - {name: "function", extraParams: map[string]any{"bad": func() {}}}, - {name: "channel", extraParams: map[string]any{"bad": make(chan struct{})}}, - {name: "struct", extraParams: map[string]any{"bad": struct{ Name string }{Name: "bad"}}}, - {name: "non string map key", extraParams: map[string]any{"bad": map[int]string{1: "one"}}}, - {name: "nan", extraParams: map[string]any{"bad": math.NaN()}}, - {name: "positive infinity", extraParams: map[string]any{"bad": math.Inf(1)}}, - {name: "negative infinity", extraParams: map[string]any{"bad": math.Inf(-1)}}, - {name: "cyclic map", extraParams: cyclicMap}, - {name: "cyclic slice", extraParams: map[string]any{"cycle": cyclicSlice}}, - {name: "malformed JSON number", extraParams: map[string]any{"value": json.Number("+1")}}, - {name: "empty nested key", extraParams: map[string]any{"nested": map[string]any{"": true}}}, - {name: "excessively deep value", extraParams: map[string]any{"nested": excessivelyDeepJSONValue()}}, - } - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}} - engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(fake)) - - _, err := engine.Run(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - Execution: &promptkit.ExecutionTargetOverride{ExtraParams: tc.extraParams}, - }) - if !errors.Is(err, promptkit.ErrInvalidRequest) { - t.Fatalf("expected ErrInvalidRequest, got %v", err) - } - if len(fake.requests) != 0 { - t.Fatalf("expected invalid request to fail before LLM call, got %d requests", len(fake.requests)) - } - }) - } -} - -func excessivelyDeepJSONValue() any { - const clearlyUnsafeContainerDepth = 1_000 - var value any = true - for level := 0; level < clearlyUnsafeContainerDepth; level++ { - if level%2 == 0 { - value = map[string]any{"child": value} - } else { - value = []any{value} - } - } - return value -} - -func excessivelyDeepStructuredOutputSchema() any { - const clearlyUnsafeSchemaNesting = 128 - schema := any(map[string]any{"type": "object"}) - for level := 0; level < clearlyUnsafeSchemaNesting; level++ { - schema = map[string]any{"allOf": []any{schema}} - } - return schema -} - -func excessivelyLargeJSONValue() any { - const clearlyUnsafeSharedOccurrences = 60_000 - shared := []any{true} - values := make([]any, clearlyUnsafeSharedOccurrences) - for i := range values { - values[i] = shared - } - return values -} - -func TestWithLLMClientRejectsNilClient(t *testing.T) { - _, err := testfixture.NewEngine(contractConfig(frameworkSchemaDir), promptkit.WithLLMClient(nil)) - if !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("expected ErrInvalidConfig, got %v", err) - } -} - -func TestNewEngineConstructsDefaultLLMClientWithoutCredentials(t *testing.T) { - if _, err := testfixture.NewEngine(contractConfig(frameworkSchemaDir)); err != nil { - t.Fatalf("expected default engine construction without credentials to succeed, got %v", err) - } -} - -func newContractEngine(t *testing.T) *promptkit.Engine { - t.Helper() - - for _, path := range []string{ - frameworkPromptDir, - frameworkProfileDir, - frameworkSchemaDir, - } { - if _, err := os.Stat(path); err != nil { - t.Fatalf("expected framework contract path %s to exist: %v", path, err) - } - } - - engine, err := testfixture.NewEngine(contractConfig(frameworkSchemaDir)) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - return engine -} - -func newContractEngineWithOptions(t *testing.T, schemaDir string, opts ...any) *promptkit.Engine { - t.Helper() - - engine, err := testfixture.NewEngine(contractConfig(schemaDir), opts...) - if err != nil { - t.Fatalf("expected engine construction to succeed, got %v", err) - } - return engine -} - -func newArtifactReaderEngine(t *testing.T, reader promptkit.ArtifactReader) *promptkit.Engine { - t.Helper() - - promptDir := t.TempDir() - writePublicPromptFile(t, promptDir, "artifact-reader", frameworkFastProfileID) - engine, err := testfixture.NewEngine(testfixture.Config{ - PromptDir: promptDir, - ProfileDir: frameworkProfileDir, - SchemaDir: frameworkSchemaDir, - }, promptkit.WithArtifactReader(reader)) - if err != nil { - t.Fatalf("construct engine with artifact reader: %v", err) - } - return engine -} - -func contractConfig(schemaDir string) testfixture.Config { - return testfixture.Config{ - PromptDir: frameworkPromptDir, - ProfileDir: frameworkProfileDir, - SchemaDir: schemaDir, - } -} - -func writePublicPromptFile(t *testing.T, dir, id, defaultProfile string) { - t.Helper() - data := `id: ` + id + ` -version: "1.0.0" -default_profile: ` + defaultProfile + ` -inputs: - - name: transcript - required: true -messages: - - role: user - content: "Summarize: {{input \"transcript\"}}" -output: - format: text - validation_mode: none - repair_attempts: 0 -` - if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(data), 0o644); err != nil { - t.Fatalf("failed to write prompt fixture: %v", err) - } -} - -func writePublicProfileFile(t *testing.T, dir, id, endpoint, model string) { - t.Helper() - data := `id: ` + id + ` -endpoint: ` + endpoint + ` -model: ` + model + ` -` - if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(data), 0o644); err != nil { - t.Fatalf("failed to write profile fixture: %v", err) - } -} - -func writePublicProfileFileWithAPIKeyEnv(t *testing.T, dir, id, endpoint, model, apiKeyEnv string) { - t.Helper() - data := `id: ` + id + ` -endpoint: ` + endpoint + ` -model: ` + model + ` -api_key_env: ` + apiKeyEnv + ` -` - if err := os.WriteFile(filepath.Join(dir, id+".yaml"), []byte(data), 0o644); err != nil { - t.Fatalf("failed to write profile fixture: %v", err) - } -} - -func publicStructuredPromptFS(id string, schemaPath string) fstest.MapFS { - return fstest.MapFS{ - "prompts/prompt.yaml": &fstest.MapFile{Data: []byte(`id: ` + id + ` -version: "1.0.0" -default_profile: contract-fast -inputs: - - name: transcript - required: true -messages: - - role: user - content: "Extract events from {{input \"transcript\"}}." -output: - format: json - validation_mode: json_schema - schema_path: ` + strconv.Quote(schemaPath) + ` - repair_attempts: 0 -`)}, - } -} - -func publicSchemaFS() fstest.MapFS { - return fstest.MapFS{ - "schemas/events.schema.json": &fstest.MapFile{Data: []byte(publicSchemaJSON())}, - } -} - -func publicSchemaJSON() string { - return `{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "required": ["events"], - "properties": { - "events": {"type": "array"} - } -}` -} - -type executionProfileFixture struct { - id string - endpoint string - model string - temperature float64 - maxTokens int - topP float64 - timeoutSeconds int - serviceTier string - reasoningEffort string - apiKeyEnv string - extraParamSource string -} - -func executionTargetFromProfileFixture(profile executionProfileFixture) promptkit.ExecutionTarget { - return promptkit.ExecutionTarget{ - Endpoint: profile.endpoint, - Model: profile.model, - Temperature: profile.temperature, - MaxTokens: profile.maxTokens, - TopP: profile.topP, - TimeoutSeconds: profile.timeoutSeconds, - ServiceTier: profile.serviceTier, - ReasoningEffort: profile.reasoningEffort, - APIKeyEnv: profile.apiKeyEnv, - ExtraParams: map[string]any{"source": profile.extraParamSource}, - } -} - -func writeExecutionProfileFixture(t *testing.T, dir string, profile executionProfileFixture) { - t.Helper() - data := fmt.Sprintf(`id: %s -endpoint: %s -model: %s -temperature: %g -max_tokens: %d -top_p: %g -timeout_seconds: %d -service_tier: %s -reasoning_effort: %s -api_key_env: %s -extra_params: - source: %q -`, - profile.id, - profile.endpoint, - profile.model, - profile.temperature, - profile.maxTokens, - profile.topP, - profile.timeoutSeconds, - profile.serviceTier, - profile.reasoningEffort, - profile.apiKeyEnv, - profile.extraParamSource, - ) - if err := os.WriteFile(filepath.Join(dir, profile.id+".yaml"), []byte(data), 0o644); err != nil { - t.Fatalf("write execution profile fixture: %v", err) - } -} - -type fakeLLMClient struct { - response *promptkit.GenerateResponse - responses []*promptkit.GenerateResponse - err error - requests []promptkit.GenerateRequest -} - -type recordingArtifactReader struct { - artifact *promptkit.Artifact - err error - refs []promptkit.ArtifactRef - read func(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) -} - -func (r *recordingArtifactReader) Read(ctx context.Context, ref promptkit.ArtifactRef) (*promptkit.Artifact, error) { - r.refs = append(r.refs, ref) - if r.read != nil { - return r.read(ctx, ref) - } - if r.err != nil { - return nil, r.err - } - return r.artifact, nil -} - -type countingSchemaFS struct { - fs.FS - mu sync.Mutex - reads map[string]int -} - -type publicDefinitionReadFailureFS struct { - fs.FS - target string - err error -} - -func (f *publicDefinitionReadFailureFS) Open(name string) (fs.File, error) { - if name == f.target { - return nil, f.err - } - return f.FS.Open(name) -} - -type cancelingPublicSchemaFS struct { - fs.FS - mu sync.Mutex - target string - cancel context.CancelFunc - canceled bool - reads int -} - -func (f *cancelingPublicSchemaFS) Open(name string) (fs.File, error) { - file, err := f.FS.Open(name) - if err != nil || name != f.target { - return file, err - } - return &cancelingPublicSchemaFile{File: file, owner: f}, nil -} - -func (f *cancelingPublicSchemaFS) readCount() int { - f.mu.Lock() - defer f.mu.Unlock() - return f.reads -} - -type cancelingPublicSchemaFile struct { - fs.File - owner *cancelingPublicSchemaFS -} - -func (f *cancelingPublicSchemaFile) Read(buffer []byte) (int, error) { - n, err := f.File.Read(buffer) - f.owner.mu.Lock() - f.owner.reads++ - if !f.owner.canceled { - f.owner.canceled = true - f.owner.cancel() - } - f.owner.mu.Unlock() - return n, err -} - -func (f *countingSchemaFS) Open(name string) (fs.File, error) { - file, err := f.FS.Open(name) - if err != nil { - return nil, err - } - return &countingSchemaFile{File: file, name: name, owner: f}, nil -} - -type countingSchemaFile struct { - fs.File - name string - owner *countingSchemaFS - counted bool -} - -func (f *countingSchemaFile) Read(buffer []byte) (int, error) { - if !f.counted { - f.counted = true - f.owner.recordRead(f.name) - } - return f.File.Read(buffer) -} - -func (f *countingSchemaFS) recordRead(name string) { - f.mu.Lock() - f.reads[name]++ - f.mu.Unlock() -} - -func (f *countingSchemaFS) readCount(name string) int { - f.mu.Lock() - defer f.mu.Unlock() - return f.reads[name] -} - -func newPublicSchemaGraphEngine( - t *testing.T, - sourceName string, - files map[string]string, - client promptkit.LLMClient, -) *promptkit.Engine { - t.Helper() - options := []any{ - testfixture.WithPromptFS(publicStructuredPromptFS("schema.graph.prompt", "root.json"), "prompts"), - testfixture.WithProfiles(testfixture.Profile{ - ID: "contract-fast", Endpoint: "http://example.test/v1", Model: "schema-model", - }), - promptkit.WithLLMClient(client), - } - config := testfixture.Config{} - - switch sourceName { - case "operating system files": - workspace := t.TempDir() - schemaRoot := filepath.Join(workspace, "schemas") - if err := os.Mkdir(schemaRoot, 0o755); err != nil { - t.Fatal(err) - } - for name, body := range files { - fileName := filepath.Clean(filepath.Join(schemaRoot, filepath.FromSlash(name))) - if err := os.MkdirAll(filepath.Dir(fileName), 0o755); err != nil { - t.Fatal(err) - } - if err := os.WriteFile(fileName, []byte(body), 0o644); err != nil { - t.Fatal(err) - } - } - config.SchemaDir = schemaRoot - case "fs.FS": - schemaFS := make(fstest.MapFS, len(files)) - for name, body := range files { - schemaFS[path.Clean(path.Join("schemas", name))] = &fstest.MapFile{Data: []byte(body)} - } - options = append(options, promptkit.WithSchemaFS(schemaFS, "schemas")) - default: - t.Fatalf("unknown schema source %q", sourceName) - } - - engine, err := testfixture.NewEngine(config, options...) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - return engine -} - -type roundTripFunc func(*http.Request) (*http.Response, error) - -func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) { - return f(req) -} - -func (f *fakeLLMClient) Generate(_ context.Context, req promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { - f.requests = append(f.requests, req) - if f.err != nil { - return nil, f.err - } - if len(f.responses) > 0 { - index := len(f.requests) - 1 - if index >= len(f.responses) { - return nil, fmt.Errorf("no response configured for generation %d", index+1) - } - return f.responses[index], nil - } - return f.response, nil -} diff --git a/errors.go b/errors.go index d0f351a..6beeb7e 100644 --- a/errors.go +++ b/errors.go @@ -72,10 +72,6 @@ func publicErrorFor(err error) error { return ErrPromptLoad case errors.Is(err, usecase.ErrProfileLoad): return ErrProfileLoad - case errors.Is(err, promptdef.ErrInvalidYAML), errors.Is(err, promptdef.ErrInvalidPromptDefinition): - return ErrPromptLoad - case isProfileLoadCause(err): - return ErrProfileLoad case errors.Is(err, usecase.ErrAPIKeyEnvMissing): return errors.Join(ErrInvalidRequest, ErrAPIKeyEnvMissing) case errors.Is(err, capacity.ErrCapacityExceeded): @@ -94,9 +90,3 @@ func publicErrorFor(err error) error { return nil } } - -func isProfileLoadCause(err error) bool { - return errors.Is(err, profile.ErrInvalidYAML) || - errors.Is(err, profile.ErrInvalidProfile) || - errors.Is(err, profile.ErrRawAPIKeyNotAllowed) -} diff --git a/generation_error_contract_test.go b/generation_error_contract_test.go index 5588f67..7d76aac 100644 --- a/generation_error_contract_test.go +++ b/generation_error_contract_test.go @@ -10,7 +10,6 @@ import ( "testing" "gitea.maximumdirect.net/eric/promptkit" - "gitea.maximumdirect.net/eric/promptkit/testfixture" ) func TestBuiltInGenerationError(t *testing.T) { @@ -22,14 +21,14 @@ func TestBuiltInGenerationError(t *testing.T) { engine := newBuiltInGenerationErrorEngine(t, http.StatusUnprocessableEntity, `{"error":{"code":"`+codeMarker+`","type":"`+typeMarker+`","message":"`+messageMarker+`"}}`) - result, err := engine.Run(context.Background(), generationErrorRunRequest()) + result, err := engine.Run(context.Background(), simpleGenerationRequest()) if result != nil { t.Fatalf("Run result = %#v, want nil", result) } assertGenerationError(t, err, http.StatusUnprocessableEntity, codeMarker, typeMarker, messageMarker) preparedEngine := newBuiltInGenerationErrorEngine(t, http.StatusServiceUnavailable, `{"error":{}}`) - prepared, err := preparedEngine.PrepareExecution(context.Background(), generationErrorRunRequest()) + prepared, err := preparedEngine.PrepareExecution(context.Background(), simpleGenerationRequest()) if err != nil { t.Fatalf("PrepareExecution: %v", err) } @@ -44,7 +43,7 @@ func TestBuiltInGenerationErrorWithAppendedMessages(t *testing.T) { const messageMarker = "combined-request-provider-marker" engine := newBuiltInGenerationErrorEngine(t, http.StatusUnprocessableEntity, `{"error":{"message":"`+messageMarker+`"}}`) - request := generationErrorRunRequest() + request := simpleGenerationRequest() request.AppendedMessages = []promptkit.RenderedMessage{ {Role: promptkit.RoleAssistant, Content: "previous response"}, {Role: promptkit.RoleUser, Content: "consumer correction"}, @@ -64,7 +63,7 @@ func TestBuiltInRepairGenerationError(t *testing.T) { messageMarker = "repair-message-marker" ) calls := 0 - config := contractConfig(frameworkSchemaDir) + config := simpleConfig("message") config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { calls++ if calls == 1 { @@ -74,11 +73,11 @@ func TestBuiltInRepairGenerationError(t *testing.T) { body := `{"error":{"code":"` + codeMarker + `","type":"` + typeMarker + `","message":"` + messageMarker + `"}}` return &http.Response{StatusCode: http.StatusUnprocessableEntity, ContentLength: int64(len(body)), Body: io.NopCloser(strings.NewReader(body))}, nil })} - engine, err := testfixture.NewEngine(config) + engine, err := promptkit.NewEngine(config) if err != nil { t.Fatalf("NewEngine: %v", err) } - req := generationErrorRunRequest() + req := simpleGenerationRequest() req.Validation = &promptkit.OutputContract{ Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, @@ -97,15 +96,15 @@ func TestBuiltInRepairGenerationError(t *testing.T) { func TestBuiltInGenerationFailureCategories(t *testing.T) { transportCause := errors.New("private transport diagnostic") - config := contractConfig(frameworkSchemaDir) + config := simpleConfig("message") config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { return nil, transportCause })} - engine, err := testfixture.NewEngine(config) + engine, err := promptkit.NewEngine(config) if err != nil { t.Fatal(err) } - result, err := engine.Run(context.Background(), generationErrorRunRequest()) + result, err := engine.Run(context.Background(), simpleGenerationRequest()) if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, promptkit.ErrTransport) || !errors.Is(err, transportCause) { t.Fatalf("transport result=%#v, error=%v", result, err) } @@ -123,7 +122,7 @@ func TestBuiltInGenerationFailureCategories(t *testing.T) { } { t.Run(tc.name, func(t *testing.T) { engine := newBuiltInGenerationErrorEngine(t, http.StatusOK, tc.body) - result, err := engine.Run(context.Background(), generationErrorRunRequest()) + result, err := engine.Run(context.Background(), simpleGenerationRequest()) if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, promptkit.ErrMalformedResponse) || errors.Is(err, promptkit.ErrTransport) { t.Fatalf("malformed result=%#v, error=%v", result, err) } @@ -140,15 +139,15 @@ func TestBuiltInGenerationPreservesContextIdentity(t *testing.T) { {name: "deadline", cause: context.DeadlineExceeded}, } { t.Run(tc.name, func(t *testing.T) { - config := contractConfig(frameworkSchemaDir) + config := simpleConfig("message") config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { return nil, tc.cause })} - engine, err := testfixture.NewEngine(config) + engine, err := promptkit.NewEngine(config) if err != nil { t.Fatal(err) } - result, err := engine.Run(context.Background(), generationErrorRunRequest()) + result, err := engine.Run(context.Background(), simpleGenerationRequest()) if result != nil || !errors.Is(err, tc.cause) || !errors.Is(err, promptkit.ErrLLMGenerate) || !errors.Is(err, promptkit.ErrTransport) { t.Fatalf("context result=%#v, error=%v", result, err) } @@ -166,8 +165,8 @@ func TestInjectedGenerationFailureCategories(t *testing.T) { {name: "arbitrary error", client: &fakeLLMClient{err: errors.New("injected failure")}}, } { t.Run(tc.name, func(t *testing.T) { - engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(tc.client)) - result, err := engine.Run(context.Background(), generationErrorRunRequest()) + engine := newSimpleEngine(t, "message", promptkit.WithLLMClient(tc.client)) + result, err := engine.Run(context.Background(), simpleGenerationRequest()) if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) || errors.Is(err, promptkit.ErrTransport) || errors.Is(err, promptkit.ErrMalformedResponse) != tc.malformed { t.Fatalf("injected result=%#v, error=%v", result, err) } @@ -208,7 +207,7 @@ func assertGenerationError(t *testing.T, err error, statusCode int, code, provid func newBuiltInGenerationErrorEngine(t *testing.T, statusCode int, body string) *promptkit.Engine { t.Helper() - config := contractConfig(frameworkSchemaDir) + config := simpleConfig("message") noRetries := 0 config.TransientRetries = &noRetries config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { @@ -218,19 +217,13 @@ func newBuiltInGenerationErrorEngine(t *testing.T, statusCode int, body string) Body: io.NopCloser(strings.NewReader(body)), }, nil })} - engine, err := testfixture.NewEngine(config) + engine, err := promptkit.NewEngine(config) if err != nil { t.Fatalf("NewEngine: %v", err) } return engine } -func generationErrorRunRequest() promptkit.RunRequest { - return promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - } +func simpleGenerationRequest() promptkit.RunRequest { + return promptkit.RunRequest{PromptID: "prompt"} } diff --git a/go.mod b/go.mod index f10f425..5e291a9 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,9 @@ go 1.25.5 require ( gitea.maximumdirect.net/eric/ConfigKit v0.5.0 - gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260926.0 gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 - gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260926.0 gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 github.com/santhosh-tekuri/jsonschema/v6 v6.0.2 - gopkg.in/yaml.v3 v3.0.1 ) require ( diff --git a/go.sum b/go.sum index 0e2a728..1c479d9 100644 --- a/go.sum +++ b/go.sum @@ -1,11 +1,7 @@ gitea.maximumdirect.net/eric/ConfigKit v0.5.0 h1:2KwG10+kApWM3avOoYA+p768xcbHUp7tQ2rqizBNpyg= gitea.maximumdirect.net/eric/ConfigKit v0.5.0/go.mod h1:tYWpsrbLyu05r8+1MapWp5fN6ZrX9I8hU2xEPedYmYk= -gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260926.0 h1:wdab+R+lgMZxzYwhbAuCrW7zdSQ1cW9ovBR6UzLmZR4= -gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v2 v2.20260926.0/go.mod h1:a3VBbh9oaGXD6ruxhtwHj+AWW6DY/SSyby1QtMmYSUo= gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0 h1:lzfCDbzJQRGzIlRPRprnAV9hVVL5zUQy80VMQ0kbYa0= gitea.maximumdirect.net/eric/promptkit-backend-openrouter/v3 v3.20260930.0/go.mod h1:HjS0f+3q0IdHCTEz+VVAEgIwIWJFAZUjuKdAtvtqKdg= -gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260926.0 h1:baCCIB6Ee5t0S4Zc++NTBIPcJEv5UPS6DlSrpZfykwg= -gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v2 v2.20260926.0/go.mod h1:mtgD1E0SGm0e/I8Ffhz+AYX7GN7RKUWJXehNFIKHs4Y= gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0 h1:h12ZTRldAmpLAomj0oQRGteFxHLN1LPE6XDl9zJNbm8= gitea.maximumdirect.net/eric/promptkit-backend-rakestrawhome/v3 v3.20260930.0/go.mod h1:OEFBeKdjTsLzqJwLmXI2O5gG3r64FktcVJVb3mBrRfg= github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI= @@ -16,7 +12,5 @@ go.yaml.in/yaml/v4 v4.0.0-rc.6 h1:1h7H1ohdUh93/FyE4YaDa1Zh64K6VVbjF4K6WUxMtH4= go.yaml.in/yaml/v4 v4.0.0-rc.6/go.mod h1:aZqd9kCMsGL7AuUv/m/PvWLdg5sjJsZ4oHDEnfPPfY0= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/backend/registry.go b/internal/backend/registry.go index f65b5df..5f452a3 100644 --- a/internal/backend/registry.go +++ b/internal/backend/registry.go @@ -5,31 +5,21 @@ package backend import ( "errors" "fmt" - "regexp" - "sort" - "strings" "gitea.maximumdirect.net/eric/promptkit/internal/domain" "gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue" - "gitea.maximumdirect.net/eric/promptkit/internal/llm" ) const ( - // OpenRouterID is the reserved ID of Promptkit's built-in OpenRouter - // backend. + // OpenRouterID identifies the maintained OpenRouter backend. OpenRouterID = "openrouter" - // RakestrawHomeID is the reserved ID of Promptkit's built-in Rakestrawhome - // backend. + // RakestrawHomeID identifies the maintained Rakestrawhome backend. RakestrawHomeID = "rakestrawhome" - - defaultQueueCapacity = 1024 ) // ErrBackendNotFound identifies a registry lookup for an unknown backend ID. var ErrBackendNotFound = errors.New("backend not found") -var environmentVariableName = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) - // Registry is an immutable collection of validated backend definitions. type Registry struct { backends map[string]domain.Backend @@ -52,36 +42,6 @@ func NewEffectiveRegistry(definitions map[string]domain.Backend) (*Registry, err return registry, nil } -// NewRegistry constructs a registry containing maintained definitions followed -// by consumer additions. Every ID must be unique across both groups. -func NewRegistry(maintained, additions []domain.Backend) (*Registry, error) { - registry := &Registry{ - backends: make(map[string]domain.Backend, len(maintained)+len(additions)), - } - - definitions := make([]domain.Backend, 0, len(maintained)+len(additions)) - definitions = append(definitions, maintained...) - definitions = append(definitions, additions...) - - for _, definition := range definitions { - definition.ID = strings.TrimSpace(definition.ID) - if definition.ID == "" { - return nil, errors.New("backend ID must not be blank") - } - if _, exists := registry.backends[definition.ID]; exists { - return nil, fmt.Errorf("backend ID %q is already registered", definition.ID) - } - - normalized, err := NormalizeDefinition(definition) - if err != nil { - return nil, err - } - registry.backends[normalized.ID] = normalized - } - - return registry, nil -} - // GetBackend returns a defensive copy of the backend registered with id. func (r *Registry) GetBackend(id string) (domain.Backend, error) { if r == nil { @@ -117,80 +77,3 @@ func (r *Registry) CapacityPolicies() map[string]domain.BackendCapacityPolicy { } return policies } - -// NormalizeDefinition validates and defensively copies one backend definition. -func NormalizeDefinition(definition domain.Backend) (domain.Backend, error) { - endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(definition.Endpoint) - if err != nil { - return domain.Backend{}, fmt.Errorf("backend %q endpoint: %w", definition.ID, err) - } - definition.Endpoint = endpoint - - definition.APIKeyEnv = strings.TrimSpace(definition.APIKeyEnv) - if definition.APIKeyEnv != "" && !environmentVariableName.MatchString(definition.APIKeyEnv) { - return domain.Backend{}, fmt.Errorf( - "backend %q api key environment variable %q is invalid", - definition.ID, - definition.APIKeyEnv, - ) - } - - if definition.ConcurrencyLimit < 0 { - return domain.Backend{}, fmt.Errorf( - "backend %q concurrency limit must not be negative", - definition.ID, - ) - } - if definition.QueueCapacity < 0 { - return domain.Backend{}, fmt.Errorf( - "backend %q queue capacity must not be negative", - definition.ID, - ) - } - if definition.ConcurrencyLimit == 0 { - if definition.QueueCapacitySet { - return domain.Backend{}, fmt.Errorf( - "backend %q queue capacity requires a positive concurrency limit", - definition.ID, - ) - } - definition.QueueCapacity = 0 - } else { - if !definition.QueueCapacitySet { - definition.QueueCapacity = defaultQueueCapacity - definition.QueueCapacitySet = true - } - maxInt := int(^uint(0) >> 1) - if definition.QueueCapacity > maxInt-definition.ConcurrencyLimit { - return domain.Backend{}, fmt.Errorf( - "backend %q total capacity overflows int", - definition.ID, - ) - } - } - - keys := make([]string, 0, len(definition.ExtraParams)) - for key := range definition.ExtraParams { - keys = append(keys, key) - } - sort.Strings(keys) - for _, key := range keys { - if key == "" { - return domain.Backend{}, fmt.Errorf("backend %q extra parameter key must not be empty", definition.ID) - } - if llm.IsReservedOpenAIChatRequestField(key) { - return domain.Backend{}, fmt.Errorf( - "backend %q extra parameter %q collides with a reserved request field", - definition.ID, - key, - ) - } - } - - extraParams, err := jsonvalue.CopyMap(definition.ExtraParams) - if err != nil { - return domain.Backend{}, fmt.Errorf("backend %q extra parameters: %w", definition.ID, err) - } - definition.ExtraParams = extraParams - return definition, nil -} diff --git a/internal/backend/registry_test.go b/internal/backend/registry_test.go index d933a0e..049db47 100644 --- a/internal/backend/registry_test.go +++ b/internal/backend/registry_test.go @@ -3,55 +3,19 @@ package backend_test import ( "errors" "reflect" - "strings" "testing" "gitea.maximumdirect.net/eric/promptkit/internal/backend" "gitea.maximumdirect.net/eric/promptkit/internal/domain" ) -const validEndpoint = "https://backend.example/v1" - -func TestRegistryIncludesMaintainedDefinitions(t *testing.T) { - maintained := []domain.Backend{ - {ID: backend.OpenRouterID, Endpoint: validEndpoint, ConcurrencyLimit: 2, QueueCapacity: 3, QueueCapacitySet: true}, - {ID: backend.RakestrawHomeID, Endpoint: "https://second.example/v1", ConcurrencyLimit: 4, QueueCapacity: 5, QueueCapacitySet: true}, - } - registry, err := backend.NewRegistry(maintained, nil) - if err != nil { - t.Fatalf("construct registry: %v", err) - } - - for _, expected := range maintained { - t.Run(expected.ID, func(t *testing.T) { - definition, err := registry.GetBackend(expected.ID) - if err != nil { - t.Fatalf("look up maintained definition: %v", err) - } - if !reflect.DeepEqual(definition, expected) { - t.Fatalf("unexpected maintained definition: %#v", definition) - } - }) - } - - policies := registry.CapacityPolicies() - if len(policies) != 2 || - policies[backend.OpenRouterID] != (domain.BackendCapacityPolicy{ - ConcurrencyLimit: 2, - QueueCapacity: 3, - }) || - policies[backend.RakestrawHomeID] != (domain.BackendCapacityPolicy{ - ConcurrencyLimit: 4, - QueueCapacity: 5, - }) { - t.Fatalf("unexpected built-in capacity policies: %#v", policies) - } -} - -func TestEffectiveRegistryKeepsLiteralIDsAndOwnsValues(t *testing.T) { +func TestEffectiveRegistryOwnsDefinitionsAndCapacityPolicies(t *testing.T) { params := map[string]any{"nested": map[string]any{"enabled": true}} registry, err := backend.NewEffectiveRegistry(map[string]domain.Backend{ - " local.v1 ": {Endpoint: validEndpoint, ExtraParams: params, ConcurrencyLimit: 2, QueueCapacitySet: true}, + " local.v1 ": { + Endpoint: "https://backend.example/v1", ExtraParams: params, + ConcurrencyLimit: 2, QueueCapacity: 3, QueueCapacitySet: true, + }, }) if err != nil { t.Fatal(err) @@ -59,341 +23,34 @@ func TestEffectiveRegistryKeepsLiteralIDsAndOwnsValues(t *testing.T) { params["nested"].(map[string]any)["enabled"] = false selected, err := registry.GetBackend(" local.v1 ") if err != nil || selected.ID != " local.v1 " || selected.ExtraParams["nested"].(map[string]any)["enabled"] != true { - t.Fatalf("effective registry changed identity or retained caller values: %+v, %v", selected, err) - } - if _, err := registry.GetBackend("local.v1"); !errors.Is(err, backend.ErrBackendNotFound) { - t.Fatalf("trimmed identity unexpectedly matched: %v", err) + t.Fatalf("effective backend was not captured under its literal ID: %+v, %v", selected, err) } selected.ExtraParams["nested"].(map[string]any)["enabled"] = false - again, _ := registry.GetBackend(" local.v1 ") - if again.ExtraParams["nested"].(map[string]any)["enabled"] != true { - t.Fatal("registry lookup exposed its stored values") + again, err := registry.GetBackend(" local.v1 ") + if err != nil || again.ExtraParams["nested"].(map[string]any)["enabled"] != true { + t.Fatalf("lookup exposed registry values: %+v, %v", again, err) } -} - -func TestRegistryNormalizesUniqueAdditionsAndIsolatesMutations(t *testing.T) { - nested := map[string]int{"limit": 2} - extraParams := map[string]any{ - "count": int64(7), - "nested": nested, - } - registry, err := backend.NewRegistry(nil, []domain.Backend{ - { - ID: " custom ", - Endpoint: " https://custom.example/openai/v1 ", - APIKeyEnv: " CUSTOM_API_KEY ", - ExtraParams: extraParams, - ConcurrencyLimit: 3, - QueueCapacity: 2, - QueueCapacitySet: true, - }, - { - ID: "Custom", - Endpoint: validEndpoint, - }, - }) - if err != nil { - t.Fatalf("construct registry: %v", err) - } - - nested["limit"] = 99 - extraParams["added"] = true - - got, err := registry.GetBackend("custom") - if err != nil { - t.Fatalf("look up custom backend: %v", err) - } - if got.ID != "custom" || - got.Endpoint != "https://custom.example/openai/v1" || - got.APIKeyEnv != "CUSTOM_API_KEY" || - got.ConcurrencyLimit != 3 || - got.QueueCapacity != 2 || - !got.QueueCapacitySet { - t.Fatalf("unexpected normalized definition: %#v", got) - } - if count, ok := got.ExtraParams["count"].(int64); !ok || count != 7 { - t.Fatalf("integer type or value changed: %#v", got.ExtraParams["count"]) - } - gotNested, ok := got.ExtraParams["nested"].(map[string]int) - if !ok || gotNested["limit"] != 2 { - t.Fatalf("container type or value changed: %#v", got.ExtraParams["nested"]) - } - if _, exists := got.ExtraParams["added"]; exists { - t.Fatalf("registry retained caller map: %#v", got.ExtraParams) - } - - gotNested["limit"] = 100 - got.ExtraParams["added"] = true - again, err := registry.GetBackend("custom") - if err != nil { - t.Fatalf("look up custom backend again: %v", err) - } - if again.ExtraParams["nested"].(map[string]int)["limit"] != 2 { - t.Fatalf("lookup exposed registry nested map: %#v", again.ExtraParams) - } - if _, exists := again.ExtraParams["added"]; exists { - t.Fatalf("lookup exposed registry map: %#v", again.ExtraParams) - } - - if _, err := registry.GetBackend("Custom"); err != nil { - t.Fatalf("backend IDs should be case-sensitive: %v", err) - } - policies := registry.CapacityPolicies() - if len(policies) != 1 { - t.Fatalf("unexpected capacity policy count: %#v", policies) - } - policies["custom"] = domain.BackendCapacityPolicy{} - delete(policies, "custom") - againPolicies := registry.CapacityPolicies() - if againPolicies["custom"] != (domain.BackendCapacityPolicy{ - ConcurrencyLimit: 3, - QueueCapacity: 2, + if !reflect.DeepEqual(policies, map[string]domain.BackendCapacityPolicy{ + " local.v1 ": {ConcurrencyLimit: 2, QueueCapacity: 3}, }) { - t.Fatalf("capacity policy map mutated registry state: %#v", againPolicies) + t.Fatalf("capacity policy = %+v", policies) + } + delete(policies, " local.v1 ") + if len(registry.CapacityPolicies()) != 1 { + t.Fatal("capacity policy map exposed registry state") } } -func TestNewRegistryNormalizesCapacityPolicy(t *testing.T) { - maxInt := int(^uint(0) >> 1) - tests := []struct { - name string - definition domain.Backend - want domain.BackendCapacityPolicy - wantSet bool - wantError bool - }{ - { - name: "unlimited when omitted", - definition: domain.Backend{}, - }, - { - name: "default queue", - definition: domain.Backend{ - ConcurrencyLimit: 2, - }, - want: domain.BackendCapacityPolicy{ - ConcurrencyLimit: 2, - QueueCapacity: 1024, - }, - wantSet: true, - }, - { - name: "explicit zero queue", - definition: domain.Backend{ - ConcurrencyLimit: 2, - QueueCapacitySet: true, - }, - want: domain.BackendCapacityPolicy{ - ConcurrencyLimit: 2, - }, - wantSet: true, - }, - { - name: "negative concurrency limit", - definition: domain.Backend{ - ConcurrencyLimit: -1, - }, - wantError: true, - }, - { - name: "negative queue capacity", - definition: domain.Backend{ - ConcurrencyLimit: 1, - QueueCapacity: -1, - QueueCapacitySet: true, - }, - wantError: true, - }, - { - name: "queue without limit", - definition: domain.Backend{ - QueueCapacitySet: true, - }, - wantError: true, - }, - { - name: "total overflow", - definition: domain.Backend{ - ConcurrencyLimit: maxInt, - QueueCapacity: 1, - QueueCapacitySet: true, - }, - wantError: true, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - tc.definition.ID = "custom" - tc.definition.Endpoint = validEndpoint - registry, err := backend.NewRegistry(nil, []domain.Backend{tc.definition}) - if tc.wantError { - if err == nil { - t.Fatal("expected invalid capacity policy error") - } - return - } - if err != nil { - t.Fatalf("construct registry: %v", err) - } - - definition, err := registry.GetBackend("custom") - if err != nil { - t.Fatalf("look up custom backend: %v", err) - } - if definition.ConcurrencyLimit != tc.want.ConcurrencyLimit || - definition.QueueCapacity != tc.want.QueueCapacity || - definition.QueueCapacitySet != tc.wantSet { - t.Fatalf("normalized capacity=(%d, %d, %t), want (%d, %d, %t)", - definition.ConcurrencyLimit, - definition.QueueCapacity, - definition.QueueCapacitySet, - tc.want.ConcurrencyLimit, - tc.want.QueueCapacity, - tc.wantSet, - ) - } - policies := registry.CapacityPolicies() - got, ok := policies["custom"] - if ok != tc.wantSet || got != tc.want { - t.Fatalf("capacity policy=(%#v, %t), want (%#v, %t)", got, ok, tc.want, tc.wantSet) - } - }) - } -} - -func TestNewRegistryRejectsDuplicateIDs(t *testing.T) { - tests := []struct { - name string - additions []domain.Backend - wantID string - }{ - { - name: "OpenRouter collision after normalization", - additions: []domain.Backend{{ID: " openrouter "}}, - wantID: backend.OpenRouterID, - }, - { - name: "Rakestrawhome collision after normalization", - additions: []domain.Backend{{ID: " rakestrawhome "}}, - wantID: backend.RakestrawHomeID, - }, - { - name: "consumer collision after normalization", - additions: []domain.Backend{ - {ID: "custom", Endpoint: validEndpoint}, - {ID: " custom ", Endpoint: "https://other.example/v1"}, - }, - wantID: "custom", - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - _, err := backend.NewRegistry(nil, tc.additions) - if err == nil { - t.Fatal("expected duplicate ID error") - } - if !strings.Contains(err.Error(), tc.wantID) { - t.Fatalf("expected error to identify %q, got %v", tc.wantID, err) - } - }) - } -} - -func TestNewRegistryValidatesIDs(t *testing.T) { - for _, id := range []string{"", " \t\n "} { - t.Run(id, func(t *testing.T) { - _, err := backend.NewRegistry(nil, []domain.Backend{{ - ID: id, - Endpoint: validEndpoint, - }}) - if err == nil { - t.Fatal("expected blank ID error") - } - }) - } -} - -func TestNewRegistryValidatesEndpoints(t *testing.T) { - tests := []struct { - name string - endpoint string - }{ - {name: "blank", endpoint: ""}, - {name: "relative", endpoint: "/v1"}, - {name: "missing host", endpoint: "https:///v1"}, - {name: "unsupported scheme", endpoint: "ftp://backend.example/v1"}, - {name: "user information", endpoint: "https://user@backend.example/v1"}, - {name: "query", endpoint: "https://backend.example/v1?mode=chat"}, - {name: "empty query", endpoint: "https://backend.example/v1?"}, - {name: "fragment", endpoint: "https://backend.example/v1#chat"}, - {name: "empty fragment", endpoint: "https://backend.example/v1#"}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - _, err := backend.NewRegistry(nil, []domain.Backend{{ - ID: "custom", - Endpoint: tc.endpoint, - }}) - if err == nil { - t.Fatal("expected invalid endpoint error") - } - }) - } -} - -func TestNewRegistryValidatesEnvironmentVariableNames(t *testing.T) { - for _, name := range []string{"1API_KEY", "API-KEY", "API KEY", "ÅPI_KEY"} { - t.Run(name, func(t *testing.T) { - _, err := backend.NewRegistry(nil, []domain.Backend{{ - ID: "custom", - Endpoint: validEndpoint, - APIKeyEnv: name, - }}) - if err == nil { - t.Fatal("expected invalid environment-variable name error") - } - }) - } -} - -func TestNewRegistryRejectsInvalidAndReservedExtraParameters(t *testing.T) { - tests := []struct { - name string - extraParams map[string]any - }{ - {name: "unsupported value", extraParams: map[string]any{"value": make(chan int)}}, - {name: "reserved key", extraParams: map[string]any{"model": "override"}}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - _, err := backend.NewRegistry(nil, []domain.Backend{{ - ID: "custom", - Endpoint: validEndpoint, - ExtraParams: tc.extraParams, - }}) - if err == nil { - t.Fatal("expected invalid extra parameters error") - } - }) - } -} - -func TestRegistryLookupReportsNotFound(t *testing.T) { - registry, err := backend.NewRegistry(nil, nil) +func TestEffectiveRegistryMissingBackend(t *testing.T) { + registry, err := backend.NewEffectiveRegistry(nil) if err != nil { - t.Fatalf("construct registry: %v", err) + t.Fatal(err) } - - _, err = registry.GetBackend("missing") - if !errors.Is(err, backend.ErrBackendNotFound) { - t.Fatalf("expected ErrBackendNotFound, got %v", err) + if _, err := registry.GetBackend("missing"); !errors.Is(err, backend.ErrBackendNotFound) { + t.Fatalf("missing backend error = %v", err) } - if !strings.Contains(err.Error(), "missing") { - t.Fatalf("expected error to identify backend, got %v", err) + if _, err := registry.GetBackend("local.v1"); !errors.Is(err, backend.ErrBackendNotFound) { + t.Fatalf("trimmed lookup unexpectedly matched: %v", err) } } diff --git a/internal/configuration/load.go b/internal/configuration/load.go index 1444433..e539e15 100644 --- a/internal/configuration/load.go +++ b/internal/configuration/load.go @@ -109,8 +109,7 @@ func Load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) return load(main, underlays) } -// load performs one ConfigKit lifecycle over the selected source tiers. Engine -// assembly is intentionally not connected to this adapter yet. +// load performs one ConfigKit lifecycle over the selected source tiers. func load(main configkit.Input, underlays []configkit.Underlay) (*loaded, error) { state := &loaded{ backends: make(map[string]map[string]fieldState), diff --git a/internal/configuration/targets.go b/internal/configuration/targets.go index a0592d5..adf34e1 100644 --- a/internal/configuration/targets.go +++ b/internal/configuration/targets.go @@ -103,7 +103,7 @@ func (state *loaded) buildExecutionProfiles() error { } target.ExtraParams = merged } - profile := domain.ExecutionProfile{ID: id, BackendID: target.BackendID, ResolvedTarget: &target, TargetPresence: presence} + profile := domain.ExecutionProfile{ID: id, ResolvedTarget: &target, TargetPresence: presence} state.indexedProfiles[id] = profile } return nil diff --git a/internal/domain/domain.go b/internal/domain/domain.go index 4522e29..c13e885 100644 --- a/internal/domain/domain.go +++ b/internal/domain/domain.go @@ -192,26 +192,11 @@ type BackendCapacityPolicy struct { QueueCapacity int } -// ExecutionProfile describes how and where to execute a model. +// ExecutionProfile holds a captured configuration target for execution. type ExecutionProfile struct { - ID string `yaml:"id"` - BaseProfileID string `yaml:"base_profile"` - BackendID string `yaml:"backend"` - Endpoint string `yaml:"endpoint"` - Model string `yaml:"model"` - Temperature float64 `yaml:"temperature"` - MaxTokens int `yaml:"max_tokens"` - TopP float64 `yaml:"top_p"` - TimeoutSeconds int `yaml:"timeout_seconds"` - ServiceTier string `yaml:"service_tier"` - ReasoningEffort string `yaml:"reasoning_effort"` - APIKeyEnv string `yaml:"api_key_env"` - APIKeyRequired bool `yaml:"-" json:"-"` - ExtraParams map[string]any `yaml:"extra_params"` - // ResolvedTarget is an execution-owned snapshot assembled from effective - // configuration and backend defaults. Legacy source repositories leave it nil. - ResolvedTarget *ExecutionTarget `yaml:"-" json:"-"` - TargetPresence ExecutionTargetPresence `yaml:"-" json:"-"` + ID string + ResolvedTarget *ExecutionTarget + TargetPresence ExecutionTargetPresence } // ExecutionTargetOverride represents per-request runtime setting overrides. diff --git a/internal/engine/engine.go b/internal/engine/engine.go index 80d5925..7f37ced 100644 --- a/internal/engine/engine.go +++ b/internal/engine/engine.go @@ -133,7 +133,7 @@ func New(cfg Config, options Options) (*Engine, error) { } return &Engine{ - runner: usecase.NewRunnerWithRepairer(snapshot, snapshot, snapshot, reader, prompt.NewCompiledRenderer(snapshot), client, validator, usecase.NewDefaultOutputRepairer(client), manager, defaultRetries), + runner: usecase.NewRunnerWithRepairer(snapshot, snapshot, reader, prompt.NewCompiledRenderer(snapshot), client, validator, usecase.NewDefaultOutputRepairer(client), manager, defaultRetries), inspection: snapshot.Inspection(), }, nil } diff --git a/internal/engine/maintained_profiles.go b/internal/engine/maintained_profiles.go deleted file mode 100644 index 619daa4..0000000 --- a/internal/engine/maintained_profiles.go +++ /dev/null @@ -1,50 +0,0 @@ -package engine - -import ( - "context" - "fmt" - - "gitea.maximumdirect.net/eric/promptkit/internal/backend" - "gitea.maximumdirect.net/eric/promptkit/internal/domain" - "gitea.maximumdirect.net/eric/promptkit/internal/profile" -) - -func maintainedProfileDefinitions() []domain.ExecutionProfile { - return []domain.ExecutionProfile{ - { - ID: "promptkit-openrouter-standard", - BaseProfileID: "gemini-flash-latest", - }, - { - ID: "promptkit-openrouter-auto", - BackendID: backend.OpenRouterID, - Model: "openrouter/auto", - }, - } -} - -func newMaintainedProfileRepository(catalogProfiles profile.Repository, maintainedBackends []domain.Backend) (profile.Repository, error) { - definitions := maintainedProfileDefinitions() - owned, err := profile.NewMemoryRepository(definitions) - if err != nil { - return nil, fmt.Errorf("create Promptkit-maintained profiles: %w", err) - } - - combined := profile.NewOverlayRepository(owned, catalogProfiles) - resolved := profile.NewResolvingRepository(combined) - registry, err := backend.NewRegistry(maintainedBackends, nil) - if err != nil { - return nil, fmt.Errorf("construct maintained backend registry: %w", err) - } - for _, definition := range definitions { - effective, err := resolved.GetProfile(context.Background(), definition.ID) - if err != nil { - return nil, fmt.Errorf("resolve Promptkit-maintained profile %q: %w", definition.ID, err) - } - if _, err := registry.GetBackend(effective.BackendID); err != nil { - return nil, fmt.Errorf("Promptkit-maintained profile %q selects unavailable backend %q: %w", definition.ID, effective.BackendID, err) - } - } - - return combined, nil -} diff --git a/internal/engine/maintained_profiles_test.go b/internal/engine/maintained_profiles_test.go deleted file mode 100644 index 413b78d..0000000 --- a/internal/engine/maintained_profiles_test.go +++ /dev/null @@ -1,78 +0,0 @@ -package engine - -import ( - "context" - "testing" - - "gitea.maximumdirect.net/eric/promptkit/internal/backend" - "gitea.maximumdirect.net/eric/promptkit/internal/domain" - "gitea.maximumdirect.net/eric/promptkit/internal/profile" -) - -func TestNewMaintainedProfileRepository(t *testing.T) { - newCatalogProfiles := func(t *testing.T, definitions []domain.ExecutionProfile) profile.Repository { - t.Helper() - repository, err := profile.NewMemoryRepository(definitions) - if err != nil { - t.Fatalf("create catalog profiles: %v", err) - } - return repository - } - newBackend := func(id string) domain.Backend { - return domain.Backend{ID: id, Endpoint: "https://" + id + ".example/v1", APIKeyEnv: "TEST_API_KEY"} - } - - t.Run("resolves inherited and direct definitions", func(t *testing.T) { - repository, err := newMaintainedProfileRepository( - newCatalogProfiles(t, []domain.ExecutionProfile{{ - ID: "gemini-flash-latest", - BackendID: backend.OpenRouterID, - Model: "catalog-selected-model", - }}), - []domain.Backend{newBackend(backend.OpenRouterID)}, - ) - if err != nil { - t.Fatalf("create maintained profiles: %v", err) - } - - resolved := profile.NewResolvingRepository(repository) - standard, err := resolved.GetProfile(context.Background(), "promptkit-openrouter-standard") - if err != nil { - t.Fatalf("resolve inherited profile: %v", err) - } - if standard.BackendID != backend.OpenRouterID || standard.Model != "catalog-selected-model" { - t.Fatalf("unexpected inherited profile: %#v", standard) - } - - auto, err := resolved.GetProfile(context.Background(), "promptkit-openrouter-auto") - if err != nil { - t.Fatalf("resolve direct profile: %v", err) - } - definitions := maintainedProfileDefinitions() - if auto.BackendID != definitions[1].BackendID || auto.Model != definitions[1].Model { - t.Fatalf("unexpected direct profile: %#v", auto) - } - }) - - t.Run("rejects a missing inherited base", func(t *testing.T) { - if _, err := newMaintainedProfileRepository( - newCatalogProfiles(t, nil), - []domain.Backend{newBackend(backend.OpenRouterID)}, - ); err == nil { - t.Fatal("expected missing base error") - } - }) - - t.Run("rejects an unavailable effective backend", func(t *testing.T) { - if _, err := newMaintainedProfileRepository( - newCatalogProfiles(t, []domain.ExecutionProfile{{ - ID: "gemini-flash-latest", - BackendID: "other", - Model: "catalog-selected-model", - }}), - []domain.Backend{newBackend("other")}, - ); err == nil { - t.Fatal("expected unavailable backend error") - } - }) -} diff --git a/internal/legacycatalog/catalog.go b/internal/legacycatalog/catalog.go deleted file mode 100644 index e800f11..0000000 --- a/internal/legacycatalog/catalog.go +++ /dev/null @@ -1,250 +0,0 @@ -// Package legacycatalog supports the existing public construction path while -// catalog configuration moves to the effective ConfigKit loader. -package legacycatalog - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "io/fs" - "path" - "sort" - "strings" - - "gitea.maximumdirect.net/eric/promptkit/internal/backend" - "gitea.maximumdirect.net/eric/promptkit/internal/domain" - "gitea.maximumdirect.net/eric/promptkit/internal/profile" -) - -// Source identifies one immutable backend catalog asset tree. -type Source struct { - Name string - ExpectedBackendID string - FS fs.FS - Root string -} - -// Set is the validated maintained backend and raw profile catalog. -type Set struct { - Backends []domain.Backend - Profiles profile.Repository - profileIDs []string -} - -// Load validates and combines immutable catalog sources in source order. -func Load(sources ...Source) (Set, error) { - if len(sources) == 0 { - return Set{}, errors.New("at least one catalog source is required") - } - loaded := Set{Backends: make([]domain.Backend, 0, len(sources))} - names := map[string]bool{} - backendIDs := map[string]bool{} - profileIDs := map[string]bool{} - for _, source := range sources { - if err := validateSource(source, names); err != nil { - return Set{}, err - } - names[source.Name] = true - definition, err := loadBackend(source) - if err != nil { - return Set{}, err - } - if backendIDs[definition.ID] { - return Set{}, fmt.Errorf("catalog %s: backend ID duplicates an earlier catalog", source.Name) - } - backendIDs[definition.ID] = true - repository, metadata, err := profile.LoadFSRepository(context.Background(), source.FS, path.Join(source.Root, "profiles")) - if err != nil { - return Set{}, fmt.Errorf("catalog %s: %w", source.Name, err) - } - if len(metadata) == 0 { - return Set{}, fmt.Errorf("catalog %s: profiles must not be empty", source.Name) - } - resolvingRepository := profile.NewResolvingRepository(repository) - for _, entry := range metadata { - if strings.HasPrefix(entry.ID, "promptkit-") { - return Set{}, fmt.Errorf("catalog %s: %s uses the reserved promptkit- profile ID prefix", source.Name, entry.Path) - } - if profileIDs[entry.ID] { - return Set{}, fmt.Errorf("catalog %s: %s duplicates an earlier profile ID", source.Name, entry.Path) - } - if containsField(entry.ExplicitFields, "endpoint") || containsField(entry.ExplicitFields, "api_key_env") { - return Set{}, fmt.Errorf("catalog %s: %s contains connection metadata", source.Name, entry.Path) - } - value, err := repository.GetProfile(context.Background(), entry.ID) - if err != nil { - return Set{}, fmt.Errorf("catalog %s: %s: %w", source.Name, entry.Path, err) - } - if err := rejectSecretKeys(value.ExtraParams); err != nil { - return Set{}, fmt.Errorf("catalog %s: %s: prohibited extra parameter key", source.Name, entry.Path) - } - resolved, err := resolvingRepository.GetProfile(context.Background(), entry.ID) - if err != nil { - return Set{}, fmt.Errorf("catalog %s: %s has invalid profile inheritance", source.Name, entry.Path) - } - if resolved.BackendID != definition.ID { - return Set{}, fmt.Errorf("catalog %s: %s selects a different backend", source.Name, entry.Path) - } - profileIDs[entry.ID] = true - loaded.profileIDs = append(loaded.profileIDs, entry.ID) - } - if loaded.Profiles == nil { - loaded.Profiles = repository - } else { - loaded.Profiles = profile.NewOverlayRepository(loaded.Profiles, repository) - } - loaded.Backends = append(loaded.Backends, definition) - } - sort.Strings(loaded.profileIDs) - return loaded, nil -} - -func validateSource(source Source, names map[string]bool) error { - if strings.TrimSpace(source.Name) == "" || source.Name != strings.TrimSpace(source.Name) { - return errors.New("catalog source name must not be blank") - } - if names[source.Name] { - return fmt.Errorf("duplicate catalog source name %q", source.Name) - } - if source.FS == nil { - return fmt.Errorf("catalog %s: filesystem is nil", source.Name) - } - if source.Root == "." || source.Root == "" || !fs.ValidPath(source.Root) { - return fmt.Errorf("catalog %s: asset root is invalid", source.Name) - } - if strings.TrimSpace(source.ExpectedBackendID) == "" { - return fmt.Errorf("catalog %s: expected backend ID is blank", source.Name) - } - return validateLayout(source) -} - -func validateLayout(source Source) error { - manifestPath := path.Join(source.Root, "backend.json") - profilesRoot := path.Join(source.Root, "profiles") - return fs.WalkDir(source.FS, source.Root, func(assetPath string, entry fs.DirEntry, err error) error { - if err != nil { - return err - } - if assetPath == source.Root { - if !entry.IsDir() { - return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath) - } - return nil - } - if entry.IsDir() { - if assetPath == profilesRoot || strings.HasPrefix(assetPath, profilesRoot+"/") { - return nil - } - return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath) - } - if assetPath == manifestPath && entry.Type().IsRegular() { - return nil - } - if strings.HasPrefix(assetPath, profilesRoot+"/") && entry.Type().IsRegular() && strings.HasSuffix(assetPath, ".yml") { - return nil - } - return fmt.Errorf("catalog %s: invalid asset path %s", source.Name, assetPath) - }) -} - -func loadBackend(source Source) (domain.Backend, error) { - data, err := fs.ReadFile(source.FS, path.Join(source.Root, "backend.json")) - if err != nil { - return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: %w", source.Name, err) - } - type manifest struct { - SchemaVersion *int `json:"schema_version"` - ID *string `json:"id"` - Endpoint *string `json:"endpoint"` - APIKeyEnv *string `json:"api_key_env"` - ConcurrencyLimit *int `json:"concurrency_limit"` - QueueCapacity *int `json:"queue_capacity"` - ExtraParams json.RawMessage `json:"extra_params"` - } - decoder := json.NewDecoder(bytes.NewReader(data)) - decoder.DisallowUnknownFields() - var value manifest - if err := decoder.Decode(&value); err != nil { - return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid manifest", source.Name) - } - var trailing any - if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { - return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid manifest", source.Name) - } - if value.SchemaVersion == nil || *value.SchemaVersion != 1 || value.ID == nil || value.Endpoint == nil || value.APIKeyEnv == nil || value.ConcurrencyLimit == nil || value.QueueCapacity == nil || value.ExtraParams == nil { - return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: required field is missing or unsupported", source.Name) - } - if *value.ID != source.ExpectedBackendID { - return domain.Backend{}, fmt.Errorf("catalog %s: backend ID does not match expected ID", source.Name) - } - if strings.TrimSpace(*value.APIKeyEnv) == "" { - return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: api key environment variable must not be blank", source.Name) - } - extraParams, err := decodeExtraParams(value.ExtraParams) - if err != nil { - return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid extra parameters", source.Name) - } - if err := rejectSecretKeys(extraParams); err != nil { - return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: prohibited extra parameter key", source.Name) - } - normalized, err := backend.NormalizeDefinition(domain.Backend{ID: *value.ID, Endpoint: *value.Endpoint, APIKeyEnv: *value.APIKeyEnv, ExtraParams: extraParams, ConcurrencyLimit: *value.ConcurrencyLimit, QueueCapacity: *value.QueueCapacity, QueueCapacitySet: true}) - if err != nil { - return domain.Backend{}, fmt.Errorf("catalog %s: backend.json: invalid backend definition", source.Name) - } - return normalized, nil -} - -func decodeExtraParams(data []byte) (map[string]any, error) { - if bytes.Equal(bytes.TrimSpace(data), []byte("null")) { - return nil, nil - } - decoder := json.NewDecoder(bytes.NewReader(data)) - decoder.UseNumber() - var value map[string]any - if err := decoder.Decode(&value); err != nil || value == nil { - return nil, errors.New("extra parameters must be an object or null") - } - var trailing any - if err := decoder.Decode(&trailing); !errors.Is(err, io.EOF) { - return nil, errors.New("extra parameters must contain exactly one JSON value") - } - return value, nil -} - -func containsField(fields []string, target string) bool { - for _, field := range fields { - if field == target { - return true - } - } - return false -} - -func rejectSecretKeys(value map[string]any) error { - return rejectSecretValue(value) -} - -func rejectSecretValue(value any) error { - switch value := value.(type) { - case map[string]any: - for key, child := range value { - switch strings.ToLower(key) { - case "api_key", "apikey", "authorization", "credential", "credentials", "password", "secret", "token", "access_token": - return errors.New("prohibited key") - } - if err := rejectSecretValue(child); err != nil { - return err - } - } - case []any: - for _, child := range value { - if err := rejectSecretValue(child); err != nil { - return err - } - } - } - return nil -} diff --git a/internal/profile/definition.go b/internal/profile/definition.go deleted file mode 100644 index fd31a68..0000000 --- a/internal/profile/definition.go +++ /dev/null @@ -1,47 +0,0 @@ -package profile - -import ( - "errors" - "strings" - - "gitea.maximumdirect.net/eric/promptkit/internal/domain" -) - -// NormalizeAndValidateDefinition normalizes and validates one source-local -// profile definition without resolving a base profile. -func NormalizeAndValidateDefinition(profile *domain.ExecutionProfile) error { - if profile == nil { - return errors.New("profile is required") - } - - profile.ID = strings.TrimSpace(profile.ID) - profile.BaseProfileID = strings.TrimSpace(profile.BaseProfileID) - profile.BackendID = strings.TrimSpace(profile.BackendID) - profile.Endpoint = strings.TrimSpace(profile.Endpoint) - - if profile.ID == "" { - return errors.New("id is required") - } - if profile.Endpoint != "" { - endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(profile.Endpoint) - if err != nil { - return err - } - profile.Endpoint = endpoint - } - if profile.BaseProfileID == "" { - if profile.BackendID == "" && profile.Endpoint == "" { - return errors.New("backend or endpoint is required") - } - if strings.TrimSpace(profile.Model) == "" { - return errors.New("model is required") - } - } - - return domain.ValidateExecutionTargetSettings(domain.ExecutionTarget{ - Temperature: profile.Temperature, - MaxTokens: profile.MaxTokens, - TopP: profile.TopP, - TimeoutSeconds: profile.TimeoutSeconds, - }) -} diff --git a/internal/profile/eager_repository.go b/internal/profile/eager_repository.go deleted file mode 100644 index acdcc40..0000000 --- a/internal/profile/eager_repository.go +++ /dev/null @@ -1,98 +0,0 @@ -package profile - -import ( - "context" - "fmt" - "io/fs" - "sort" - "strings" - - "gitea.maximumdirect.net/eric/promptkit/internal/domain" - "gitea.maximumdirect.net/eric/promptkit/internal/filecatalog" - "gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue" -) - -// LoadedProfileMetadata identifies one profile accepted by LoadFSRepository. -type LoadedProfileMetadata struct { - // ID is the normalized profile ID. - ID string - // Path is the safe root-relative source path. - Path string - // ExplicitFields lists the sorted top-level YAML fields present in source. - ExplicitFields []string -} - -// LoadFSRepository eagerly validates every profile under root and returns an -// immutable raw repository and independently owned source metadata. -func LoadFSRepository(ctx context.Context, fsys fs.FS, root string) (Repository, []LoadedProfileMetadata, error) { - if fsys == nil { - return nil, nil, fmt.Errorf("failed to read profile directory: filesystem is nil") - } - paths, err := filecatalog.FindFSYAMLFiles(ctx, fsys, root) - if err != nil { - return nil, nil, fmt.Errorf("failed to read profile directory: %w", err) - } - - repository := &loadedRepository{profiles: make(map[string]domain.ExecutionProfile, len(paths))} - metadata := make([]LoadedProfileMetadata, 0, len(paths)) - for _, path := range paths { - if err := ctx.Err(); err != nil { - return nil, nil, err - } - data, err := fs.ReadFile(fsys, path) - if err != nil { - return nil, nil, fmt.Errorf("failed to read profile file %s: %w", filecatalog.DisplayPath(root, path), err) - } - fileMetadata, err := readProfileFileMetadata(data) - if err != nil { - return nil, nil, fmt.Errorf("%w: %s", ErrInvalidYAML, filecatalog.DisplayPath(root, path)) - } - if fileMetadata.hasRawAPIKey { - return nil, nil, fmt.Errorf("%w: %s", ErrRawAPIKeyNotAllowed, filecatalog.DisplayPath(root, path)) - } - definition, err := decodeProfile(data) - if err != nil { - return nil, nil, fmt.Errorf("%w: %s", ErrInvalidYAML, filecatalog.DisplayPath(root, path)) - } - definition.ExtraParams, err = jsonvalue.CopyMap(definition.ExtraParams) - if err != nil { - return nil, nil, fmt.Errorf("%w: %s", ErrInvalidProfile, filecatalog.DisplayPath(root, path)) - } - if err := NormalizeAndValidateDefinition(definition); err != nil { - return nil, nil, fmt.Errorf("%w: %s", ErrInvalidProfile, filecatalog.DisplayPath(root, path)) - } - if _, exists := repository.profiles[definition.ID]; exists { - return nil, nil, fmt.Errorf("%w: %s: duplicate profile ID", ErrInvalidProfile, filecatalog.DisplayPath(root, path)) - } - repository.profiles[definition.ID] = *definition - fields := append([]string(nil), fileMetadata.explicitFields...) - sort.Strings(fields) - metadata = append(metadata, LoadedProfileMetadata{ - ID: definition.ID, - Path: filecatalog.DisplayPath(root, path), - ExplicitFields: fields, - }) - } - sort.Slice(metadata, func(left, right int) bool { return metadata[left].ID < metadata[right].ID }) - return repository, metadata, nil -} - -type loadedRepository struct { - profiles map[string]domain.ExecutionProfile -} - -func (r *loadedRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) { - if err := ctx.Err(); err != nil { - return nil, err - } - definition, found := r.profiles[strings.TrimSpace(id)] - if !found { - return nil, ErrProfileNotFound - } - extraParams, err := jsonvalue.CopyMap(definition.ExtraParams) - if err != nil { - return nil, fmt.Errorf("copy loaded profile %q: %w", definition.ID, err) - } - definition.ExtraParams = extraParams - return &definition, nil -} diff --git a/internal/profile/eager_repository_test.go b/internal/profile/eager_repository_test.go deleted file mode 100644 index 0b5ba60..0000000 --- a/internal/profile/eager_repository_test.go +++ /dev/null @@ -1,110 +0,0 @@ -package profile - -import ( - "context" - "errors" - "io/fs" - "reflect" - "testing" - "testing/fstest" -) - -func TestLoadFSRepositoryLoadsSortedIndependentProfiles(t *testing.T) { - fsys := fstest.MapFS{ - "profiles/z.yml": {Data: []byte("id: z\nbackend: local\nmodel: z-model\nextra_params:\n nested:\n value: one\n")}, - "profiles/a.yml": {Data: []byte("id: a\nbackend: local\nmodel: a-model\nendpoint: ''\n")}, - } - - repository, metadata, err := LoadFSRepository(context.Background(), fsys, "profiles") - if err != nil { - t.Fatalf("load repository: %v", err) - } - if len(metadata) != 2 || metadata[0].ID != "a" || metadata[1].ID != "z" || metadata[0].Path != "a.yml" { - t.Fatalf("unexpected metadata: %#v", metadata) - } - if len(metadata[0].ExplicitFields) != 4 || metadata[0].ExplicitFields[0] != "backend" || metadata[0].ExplicitFields[1] != "endpoint" { - t.Fatalf("expected explicitly empty endpoint metadata, got %#v", metadata[0].ExplicitFields) - } - metadata[1].ExplicitFields[0] = "changed" - first, err := repository.GetProfile(context.Background(), "z") - if err != nil { - t.Fatalf("load profile: %v", err) - } - first.ExtraParams["nested"].(map[string]any)["value"] = "changed" - second, err := repository.GetProfile(context.Background(), "z") - if err != nil { - t.Fatalf("reload profile: %v", err) - } - if second.ExtraParams["nested"].(map[string]any)["value"] != "one" { - t.Fatalf("profile value was not defensively copied: %#v", second) - } -} - -func TestLoadFSRepositoryRejectsInvalidProfiles(t *testing.T) { - tests := map[string]fstest.MapFS{ - "duplicate IDs": { - "profiles/one.yml": {Data: []byte("id: duplicate\nbackend: local\nmodel: one\n")}, - "profiles/two.yml": {Data: []byte("id: duplicate\nbackend: local\nmodel: two\n")}, - }, - "raw API key": { - "profiles/one.yml": {Data: []byte("id: one\nbackend: local\nmodel: one\napi_key: forbidden\n")}, - }, - "multiple documents": { - "profiles/one.yml": {Data: []byte("id: one\nbackend: local\nmodel: one\n---\nid: two\n")}, - }, - } - for name, fsys := range tests { - t.Run(name, func(t *testing.T) { - _, _, err := LoadFSRepository(context.Background(), fsys, "profiles") - if err == nil { - t.Fatal("expected load failure") - } - }) - } -} - -func TestLoadedRepositoryHonorsCancellationAndMissingProfiles(t *testing.T) { - repository, _, err := LoadFSRepository(context.Background(), fstest.MapFS{ - "profiles/one.yml": {Data: []byte("id: one\nbackend: local\nmodel: one\n")}, - }, "profiles") - if err != nil { - t.Fatalf("load repository: %v", err) - } - ctx, cancel := context.WithCancel(context.Background()) - cancel() - if _, err := repository.GetProfile(ctx, "one"); !errors.Is(err, context.Canceled) { - t.Fatalf("expected cancellation, got %v", err) - } - if _, err := repository.GetProfile(context.Background(), "missing"); !errors.Is(err, ErrProfileNotFound) { - t.Fatalf("expected missing profile, got %v", err) - } -} - -func TestLoadFSRepositoryMatchesPointLookupForRawProfiles(t *testing.T) { - fsys := fstest.MapFS{ - "profiles/base.yml": {Data: []byte("id: base\nbackend: local\nmodel: base-model\n")}, - "profiles/derived.yml": {Data: []byte("id: derived\nbase_profile: base\nreasoning_effort: high\n")}, - } - eager, _, err := LoadFSRepository(context.Background(), fsys, "profiles") - if err != nil { - t.Fatalf("load eager repository: %v", err) - } - pointLookup := NewFSRepository(fsys, "profiles") - for _, id := range []string{"base", "derived"} { - t.Run(id, func(t *testing.T) { - got, err := eager.GetProfile(context.Background(), id) - if err != nil { - t.Fatalf("load eager profile: %v", err) - } - want, err := pointLookup.GetProfile(context.Background(), id) - if err != nil { - t.Fatalf("load point-in-time profile: %v", err) - } - if !reflect.DeepEqual(got, want) { - t.Fatalf("raw profiles differ: got %#v, want %#v", got, want) - } - }) - } -} - -var _ fs.FS = fstest.MapFS{} diff --git a/internal/profile/filesystem_repository.go b/internal/profile/filesystem_repository.go deleted file mode 100644 index a93d79b..0000000 --- a/internal/profile/filesystem_repository.go +++ /dev/null @@ -1,259 +0,0 @@ -package profile - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "io/fs" - "os" - "strings" - - "gitea.maximumdirect.net/eric/promptkit/internal/domain" - "gitea.maximumdirect.net/eric/promptkit/internal/filecatalog" - "gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue" - "gopkg.in/yaml.v3" -) - -var ( - ErrProfileNotFound = errors.New("execution profile not found") - ErrInvalidYAML = errors.New("invalid YAML format") - ErrInvalidProfile = errors.New("invalid execution profile configuration") - ErrRawAPIKeyNotAllowed = errors.New("raw api_key is not allowed; use api_key_env") -) - -type filesystemRepository struct { - dir string -} - -func NewFilesystemRepository(dir string) Repository { - return &filesystemRepository{dir: dir} -} - -func (r *filesystemRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) { - return loadProfile(ctx, os.DirFS(r.dir), ".", id) -} - -type fsRepository struct { - fsys fs.FS - root string -} - -func NewFSRepository(fsys fs.FS, root string) Repository { - return &fsRepository{fsys: fsys, root: root} -} - -func (r *fsRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) { - return loadProfile(ctx, r.fsys, r.root, id) -} - -type overlayRepository struct { - primary Repository - fallback Repository -} - -func NewOverlayRepository(primary, fallback Repository) Repository { - return &overlayRepository{primary: primary, fallback: fallback} -} - -func (r *overlayRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) { - if r.primary != nil { - prof, err := r.primary.GetProfile(ctx, id) - if err == nil { - return prof, nil - } - if !errors.Is(err, ErrProfileNotFound) { - return nil, err - } - } - if r.fallback == nil { - return nil, ErrProfileNotFound - } - return r.fallback.GetProfile(ctx, id) -} - -func loadProfile(ctx context.Context, fsys fs.FS, root string, id string) (*domain.ExecutionProfile, error) { - id = strings.TrimSpace(id) - if id == "" { - return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile) - } - if fsys == nil { - return nil, fmt.Errorf("failed to read profile directory: filesystem is nil") - } - - files, err := filecatalog.FindFSYAMLFiles(ctx, fsys, root) - if err != nil { - return nil, fmt.Errorf("failed to read profile directory: %w", err) - } - - var matches []profileMatch - for _, fullPath := range files { - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - - relPath := filecatalog.DisplayPath(root, fullPath) - data, err := fs.ReadFile(fsys, fullPath) - if err != nil { - return nil, fmt.Errorf("failed to read profile file %s: %w", relPath, err) - } - metadata, metadataErr := readProfileFileMetadata(data) - idMatch := metadata.matchesID(id) - if metadataErr != nil { - if idMatch { - return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, metadataErr) - } - continue - } - if metadata.hasRawAPIKey { - if idMatch { - return nil, fmt.Errorf("%w: %s", ErrRawAPIKeyNotAllowed, relPath) - } - continue - } - if !idMatch { - continue - } - - prof, err := decodeProfile(data) - if err != nil { - return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err) - } - - prof.ID = strings.TrimSpace(prof.ID) - if prof.ID != id { - continue - } - prof.ExtraParams, err = jsonvalue.CopyMap(prof.ExtraParams) - if err != nil { - return nil, fmt.Errorf("%w: %s: %v", ErrInvalidProfile, relPath, err) - } - if err := NormalizeAndValidateDefinition(prof); err != nil { - if errors.Is(err, ErrRawAPIKeyNotAllowed) { - return nil, fmt.Errorf("%w: %s", err, relPath) - } - return nil, fmt.Errorf("%w: %s: %v", ErrInvalidProfile, relPath, err) - } - matches = append(matches, profileMatch{ - profile: prof, - path: relPath, - }) - } - - if len(matches) > 1 { - paths := make([]string, 0, len(matches)) - for _, match := range matches { - paths = append(paths, match.path) - } - return nil, fmt.Errorf("%w: duplicate execution profile id %q found in: %s", ErrInvalidProfile, id, strings.Join(paths, ", ")) - } - - if len(matches) == 1 { - return matches[0].profile, nil - } - - return nil, ErrProfileNotFound -} - -type profileMatch struct { - profile *domain.ExecutionProfile - path string -} - -type profileFileMetadata struct { - ids []string - hasRawAPIKey bool - explicitFields []string -} - -func readProfileFileMetadata(data []byte) (profileFileMetadata, error) { - decoder := yaml.NewDecoder(bytes.NewReader(data)) - var node yaml.Node - if err := decoder.Decode(&node); err != nil { - return profileFileMetadata{}, err - } - metadata := profileMetadataFromNode(&node) - documentCount := 1 - for { - var trailing yaml.Node - err := decoder.Decode(&trailing) - if errors.Is(err, io.EOF) { - if documentCount == 1 { - return metadata, nil - } - return metadata, errors.New("profile file must contain exactly one YAML document") - } - if err != nil { - return metadata, err - } - documentCount++ - metadata.merge(profileMetadataFromNode(&trailing)) - } -} - -func profileMetadataFromNode(node *yaml.Node) profileFileMetadata { - if node.Kind != yaml.DocumentNode || len(node.Content) == 0 { - return profileFileMetadata{} - } - mapping := node.Content[0] - if mapping.Kind != yaml.MappingNode { - return profileFileMetadata{} - } - - var metadata profileFileMetadata - for i := 0; i+1 < len(mapping.Content); i += 2 { - key := mapping.Content[i] - value := mapping.Content[i+1] - metadata.explicitFields = append(metadata.explicitFields, key.Value) - switch key.Value { - case "id": - metadata.ids = append(metadata.ids, strings.TrimSpace(value.Value)) - case "api_key": - metadata.hasRawAPIKey = true - } - } - return metadata -} - -func (m profileFileMetadata) matchesID(id string) bool { - for _, candidate := range m.ids { - if candidate == id { - return true - } - } - return false -} - -func (m *profileFileMetadata) merge(other profileFileMetadata) { - m.ids = append(m.ids, other.ids...) - m.hasRawAPIKey = m.hasRawAPIKey || other.hasRawAPIKey - m.explicitFields = append(m.explicitFields, other.explicitFields...) -} - -func decodeProfile(data []byte) (*domain.ExecutionProfile, error) { - var prof domain.ExecutionProfile - decoder := yaml.NewDecoder(bytes.NewReader(data)) - decoder.KnownFields(true) - if err := decoder.Decode(&prof); err != nil { - return nil, err - } - if err := requireYAMLStreamEnd(decoder); err != nil { - return nil, err - } - return &prof, nil -} - -func requireYAMLStreamEnd(decoder *yaml.Decoder) error { - var trailing yaml.Node - err := decoder.Decode(&trailing) - if errors.Is(err, io.EOF) { - return nil - } - if err != nil { - return err - } - return errors.New("profile file must contain exactly one YAML document") -} diff --git a/internal/profile/memory_repository.go b/internal/profile/memory_repository.go deleted file mode 100644 index 67fb0de..0000000 --- a/internal/profile/memory_repository.go +++ /dev/null @@ -1,50 +0,0 @@ -package profile - -import ( - "context" - "fmt" - - "gitea.maximumdirect.net/eric/promptkit/internal/domain" - "gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue" -) - -// NewMemoryRepository constructs an immutable repository from profile -// definitions supplied by an internal caller. -func NewMemoryRepository(definitions []domain.ExecutionProfile) (Repository, error) { - repository := &memoryRepository{profiles: make(map[string]domain.ExecutionProfile, len(definitions))} - for _, definition := range definitions { - extraParams, err := jsonvalue.CopyMap(definition.ExtraParams) - if err != nil { - return nil, err - } - definition.ExtraParams = extraParams - if err := NormalizeAndValidateDefinition(&definition); err != nil { - return nil, err - } - if _, exists := repository.profiles[definition.ID]; exists { - return nil, fmt.Errorf("duplicate profile id %q", definition.ID) - } - repository.profiles[definition.ID] = definition - } - return repository, nil -} - -type memoryRepository struct { - profiles map[string]domain.ExecutionProfile -} - -func (r *memoryRepository) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) { - if r == nil { - return nil, ErrProfileNotFound - } - definition, found := r.profiles[id] - if !found { - return nil, ErrProfileNotFound - } - extraParams, err := jsonvalue.CopyMap(definition.ExtraParams) - if err != nil { - return nil, fmt.Errorf("copy memory profile %q: %w", definition.ID, err) - } - definition.ExtraParams = extraParams - return &definition, nil -} diff --git a/internal/profile/memory_repository_test.go b/internal/profile/memory_repository_test.go deleted file mode 100644 index edd3c01..0000000 --- a/internal/profile/memory_repository_test.go +++ /dev/null @@ -1,91 +0,0 @@ -package profile - -import ( - "context" - "errors" - "testing" - - "gitea.maximumdirect.net/eric/promptkit/internal/domain" -) - -func TestMemoryRepositoryOwnsNormalizedDefinitions(t *testing.T) { - nested := map[string]any{"value": "original"} - definitions := []domain.ExecutionProfile{{ - ID: " profile ", - BackendID: " backend ", - Endpoint: " https://example.test/v1/ ", - Model: "model", - ExtraParams: map[string]any{"nested": nested}, - }} - - repository, err := NewMemoryRepository(definitions) - if err != nil { - t.Fatalf("construct repository: %v", err) - } - nested["value"] = "mutated input" - - first, err := repository.GetProfile(context.Background(), "profile") - if err != nil { - t.Fatalf("load profile: %v", err) - } - if first.ID != "profile" || first.BackendID != "backend" || first.Endpoint != "https://example.test/v1/" { - t.Fatalf("normalized definition = %#v", first) - } - if first.ExtraParams["nested"].(map[string]any)["value"] != "original" { - t.Fatalf("repository retained input mutation: %#v", first.ExtraParams) - } - first.ExtraParams["nested"].(map[string]any)["value"] = "mutated result" - - second, err := repository.GetProfile(context.Background(), "profile") - if err != nil { - t.Fatalf("reload profile: %v", err) - } - if second.ExtraParams["nested"].(map[string]any)["value"] != "original" { - t.Fatalf("repository retained result mutation: %#v", second.ExtraParams) - } -} - -func TestMemoryRepositoryPreservesInheritanceAndRejectsInvalidDefinitions(t *testing.T) { - derived, err := NewMemoryRepository([]domain.ExecutionProfile{{ - ID: "derived", - BaseProfileID: "base", - ReasoningEffort: "high", - }}) - if err != nil { - t.Fatalf("construct derived repository: %v", err) - } - profile, err := derived.GetProfile(context.Background(), "derived") - if err != nil { - t.Fatalf("load derived profile: %v", err) - } - if profile.BaseProfileID != "base" || profile.ReasoningEffort != "high" { - t.Fatalf("derived profile = %#v", profile) - } - if _, err := derived.GetProfile(context.Background(), "missing"); !errors.Is(err, ErrProfileNotFound) { - t.Fatalf("missing profile error = %v, want ErrProfileNotFound", err) - } - - tests := []struct { - name string - definitions []domain.ExecutionProfile - }{ - { - name: "normalized duplicate IDs", - definitions: []domain.ExecutionProfile{ - {ID: " duplicate ", Endpoint: "https://one.example/v1", Model: "one"}, - {ID: "duplicate", Endpoint: "https://two.example/v1", Model: "two"}, - }, - }, - { - name: "invalid extra parameters", - definitions: []domain.ExecutionProfile{{ID: "invalid", Endpoint: "https://example.test/v1", Model: "model", ExtraParams: map[string]any{"invalid": func() {}}}}, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - if _, err := NewMemoryRepository(test.definitions); err == nil { - t.Fatal("expected repository construction error") - } - }) - } -} diff --git a/internal/profile/repository.go b/internal/profile/repository.go index af82744..cc17164 100644 --- a/internal/profile/repository.go +++ b/internal/profile/repository.go @@ -2,11 +2,15 @@ package profile import ( "context" + "errors" "gitea.maximumdirect.net/eric/promptkit/internal/domain" ) -// Repository loads execution profiles. +// ErrProfileNotFound identifies an absent profile in the captured index. +var ErrProfileNotFound = errors.New("execution profile not found") + +// Repository selects execution profiles from a captured index. type Repository interface { GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) } diff --git a/internal/profile/repository_benchmark_test.go b/internal/profile/repository_benchmark_test.go deleted file mode 100644 index 41cc6fd..0000000 --- a/internal/profile/repository_benchmark_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package profile - -import ( - "context" - "fmt" - "testing" - "testing/fstest" -) - -func BenchmarkProfileRepositoryLookup(b *testing.B) { - for _, size := range []int{10, 1000} { - b.Run(fmt.Sprintf("catalog-%d", size), func(b *testing.B) { - files := fstest.MapFS{ - "target.yaml": profileMapFile(` -id: target -endpoint: http://localhost:8000/v1 -model: target-model -extra_params: - selected: true -`), - } - metadataNames := []string{"target.yaml"} - for i := 1; i < size; i++ { - name := fmt.Sprintf("profile-%04d.yaml", i) - files[name] = profileMapFile(fmt.Sprintf(` -id: profile-%04d -endpoint: http://localhost:8000/v1 -model: unrelated-model -temperature: 0.5 -max_tokens: 500 -extra_params: - provider: - order: - - first - - second -`, i)) - metadataNames = append(metadataNames, name) - } - - fsys := &recordingProfileFS{FS: files} - repo := NewFSRepository(fsys, ".") - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - if _, err := repo.GetProfile(context.Background(), "target"); err != nil { - b.Fatal(err) - } - } - b.StopTimer() - - for _, name := range metadataNames { - if got := fsys.openCount(name); got != b.N { - b.Fatalf("metadata %q opens = %d, want %d", name, got, b.N) - } - } - }) - } -} diff --git a/internal/profile/repository_test.go b/internal/profile/repository_test.go deleted file mode 100644 index aabed53..0000000 --- a/internal/profile/repository_test.go +++ /dev/null @@ -1,1140 +0,0 @@ -package profile - -import ( - "context" - "errors" - "fmt" - "io/fs" - "os" - "path/filepath" - "strings" - "sync" - "testing" - "testing/fstest" - - "gitea.maximumdirect.net/eric/promptkit/internal/domain" -) - -func TestFilesystemRepository_GetProfile(t *testing.T) { - tmpDir, err := os.MkdirTemp("", "execution_profile_test") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(tmpDir) - - files, err := os.ReadDir("testdata") - if err != nil { - t.Fatalf("failed to read testdata: %v", err) - } - for _, f := range files { - src := filepath.Join("testdata", f.Name()) - dst := filepath.Join(tmpDir, f.Name()) - data, err := os.ReadFile(src) - if err != nil { - t.Fatal(err) - } - if err := os.WriteFile(dst, data, 0644); err != nil { - t.Fatal(err) - } - } - - repo := NewFilesystemRepository(tmpDir) - ctx := context.Background() - - t.Run("valid local profile", func(t *testing.T) { - p, err := repo.GetProfile(ctx, "local-default") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if p.ID != "local-default" { - t.Fatalf("unexpected id: %q", p.ID) - } - if p.Endpoint == "" || p.Model == "" { - t.Fatalf("expected endpoint/model to be set: %+v", p) - } - }) - - t.Run("backend and endpoint connection matrix", func(t *testing.T) { - tests := []struct { - name string - connection string - wantBackend string - wantEndpoint string - wantErr bool - }{ - {name: "backend only", connection: "backend: ' openrouter '", wantBackend: "openrouter"}, - {name: "endpoint only", connection: "endpoint: ' https://localhost:8000/nested/v1 '", wantEndpoint: "https://localhost:8000/nested/v1"}, - {name: "both", connection: "backend: openrouter\nendpoint: http://localhost:8000/v1", wantBackend: "openrouter", wantEndpoint: "http://localhost:8000/v1"}, - {name: "neither", wantErr: true}, - {name: "blank backend", connection: "backend: ' '", wantErr: true}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - id := "connection-" + strings.ReplaceAll(tt.name, " ", "-") - writeProfileTestFile(t, filepath.Join(tmpDir, id+".yaml"), "id: "+id+"\nmodel: model\n"+tt.connection+"\n") - - p, err := repo.GetProfile(ctx, id) - if tt.wantErr { - if !errors.Is(err, ErrInvalidProfile) { - t.Fatalf("expected ErrInvalidProfile, got %v", err) - } - return - } - if err != nil { - t.Fatalf("expected profile to load, got %v", err) - } - if p.BackendID != tt.wantBackend || p.Endpoint != tt.wantEndpoint { - t.Fatalf("unexpected connection values: backend=%q endpoint=%q", p.BackendID, p.Endpoint) - } - }) - } - }) - - t.Run("valid profile with api_key_env", func(t *testing.T) { - p, err := repo.GetProfile(ctx, "local-secure") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if p.APIKeyEnv != "PROMPTKIT_API_KEY" { - t.Fatalf("unexpected api_key_env: %q", p.APIKeyEnv) - } - if p.ReasoningEffort != "medium" { - t.Fatalf("unexpected reasoning_effort: %q", p.ReasoningEffort) - } - if p.ServiceTier != "priority" { - t.Fatalf("unexpected service_tier: %q", p.ServiceTier) - } - }) - - t.Run("valid nested profile", func(t *testing.T) { - nestedDir := filepath.Join(tmpDir, "local") - if err := os.MkdirAll(nestedDir, 0o755); err != nil { - t.Fatal(err) - } - writeProfileTestFile(t, filepath.Join(nestedDir, "nested-local.yaml"), ` -id: nested-local -endpoint: http://localhost:8000/v1 -model: nested-model -temperature: 0.1 -`) - - p, err := repo.GetProfile(ctx, "nested-local") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if p.Model != "nested-model" { - t.Fatalf("unexpected model: %q", p.Model) - } - }) - - t.Run("duplicate profile IDs fail as ambiguous", func(t *testing.T) { - writeProfileTestFile(t, filepath.Join(tmpDir, "duplicate-profile-a.yaml"), ` -id: duplicate-profile -endpoint: http://localhost:8000/v1 -model: first-model -`) - nestedDir := filepath.Join(tmpDir, "duplicates") - if err := os.MkdirAll(nestedDir, 0o755); err != nil { - t.Fatal(err) - } - writeProfileTestFile(t, filepath.Join(nestedDir, "duplicate-profile-b.yaml"), ` -id: duplicate-profile -endpoint: http://localhost:8000/v1 -model: second-model -`) - - _, err := repo.GetProfile(ctx, "duplicate-profile") - if !errors.Is(err, ErrInvalidProfile) { - t.Fatalf("expected duplicate profile to return ErrInvalidProfile, got %v", err) - } - for _, want := range []string{"duplicate execution profile id", "duplicate-profile-a.yaml", filepath.Join("duplicates", "duplicate-profile-b.yaml")} { - if !strings.Contains(err.Error(), want) { - t.Fatalf("expected error to contain %q, got %v", want, err) - } - } - }) - - t.Run("nested raw api_key rejected for likely target file", func(t *testing.T) { - nestedDir := filepath.Join(tmpDir, "secure") - if err := os.MkdirAll(nestedDir, 0o755); err != nil { - t.Fatal(err) - } - writeProfileTestFile(t, filepath.Join(nestedDir, "not_named_like_id.yaml"), ` -id: nested_raw_api_key -endpoint: http://localhost:8000/v1 -model: m -api_key: secret -`) - - _, err := repo.GetProfile(ctx, "nested_raw_api_key") - if !errors.Is(err, ErrRawAPIKeyNotAllowed) { - t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err) - } - if !strings.Contains(err.Error(), filepath.Join("secure", "not_named_like_id.yaml")) { - t.Fatalf("expected nested path in error, got %v", err) - } - }) - - t.Run("raw api_key in non-target profile is ignored", func(t *testing.T) { - writeProfileTestFile(t, filepath.Join(tmpDir, "raw-api-key-non-target.yaml"), ` -id: raw-api-key-non-target -endpoint: http://localhost:8000/v1 -model: m -api_key: secret -`) - - _, err := repo.GetProfile(ctx, "does-not-exist-with-raw-key-nearby") - if !errors.Is(err, ErrProfileNotFound) { - t.Fatalf("expected ErrProfileNotFound for non-target raw api_key file, got %v", err) - } - }) - - t.Run("unidentifiable invalid yaml is unrelated", func(t *testing.T) { - _, err := repo.GetProfile(ctx, "invalid_yaml") - if !errors.Is(err, ErrProfileNotFound) { - t.Fatalf("expected ErrProfileNotFound, got %v", err) - } - }) - - t.Run("missing id", func(t *testing.T) { - _, err := repo.GetProfile(ctx, "missing_id") - if !errors.Is(err, ErrProfileNotFound) { - t.Fatalf("expected ErrProfileNotFound, got %v", err) - } - }) - - t.Run("missing endpoint", func(t *testing.T) { - _, err := repo.GetProfile(ctx, "missing-endpoint") - if !errors.Is(err, ErrInvalidProfile) { - t.Fatalf("expected ErrInvalidProfile, got %v", err) - } - }) - - t.Run("missing model", func(t *testing.T) { - _, err := repo.GetProfile(ctx, "missing-model") - if !errors.Is(err, ErrInvalidProfile) { - t.Fatalf("expected ErrInvalidProfile, got %v", err) - } - }) - - t.Run("unknown field", func(t *testing.T) { - _, err := repo.GetProfile(ctx, "unknown-field") - if !errors.Is(err, ErrInvalidYAML) { - t.Fatalf("expected ErrInvalidYAML for strict decode unknown field, got %v", err) - } - }) - - t.Run("raw api_key rejected", func(t *testing.T) { - _, err := repo.GetProfile(ctx, "raw-api-key") - if !errors.Is(err, ErrRawAPIKeyNotAllowed) { - t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err) - } - }) - - t.Run("profile not found", func(t *testing.T) { - _, err := repo.GetProfile(ctx, "does-not-exist") - if !errors.Is(err, ErrProfileNotFound) { - t.Fatalf("expected ErrProfileNotFound, got %v", err) - } - }) -} - -func writeProfileTestFile(t *testing.T, path string, content string) { - t.Helper() - if err := os.WriteFile(path, []byte(strings.TrimLeft(content, "\n")), 0o644); err != nil { - t.Fatalf("failed to write profile test file %q: %v", path, err) - } -} - -func TestFSRepository(t *testing.T) { - ctx := context.Background() - - t.Run("loads valid profiles from nested directories", func(t *testing.T) { - repo := NewFSRepository(fstest.MapFS{ - "profiles/provider/nested.yaml": profileMapFile(` -id: nested-profile -endpoint: http://localhost:8000/v1 -model: nested-model -temperature: 0.1 -`), - }, "profiles") - - p, err := repo.GetProfile(ctx, "nested-profile") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if p.ID != "nested-profile" || p.Model != "nested-model" { - t.Fatalf("unexpected profile: %+v", p) - } - }) - - t.Run("rejects unknown YAML fields", func(t *testing.T) { - repo := NewFSRepository(fstest.MapFS{ - "profiles/unknown.yaml": profileMapFile(` -id: unknown-profile -endpoint: http://localhost:8000/v1 -model: model -unknown: value -`), - }, "profiles") - - _, err := repo.GetProfile(ctx, "unknown-profile") - if !errors.Is(err, ErrInvalidYAML) { - t.Fatalf("expected ErrInvalidYAML, got %v", err) - } - }) - - t.Run("rejects raw api_key in selected profile", func(t *testing.T) { - repo := NewFSRepository(fstest.MapFS{ - "profiles/raw.yaml": profileMapFile(` -id: raw-profile -endpoint: http://localhost:8000/v1 -model: model -api_key: secret -`), - }, "profiles") - - _, err := repo.GetProfile(ctx, "raw-profile") - if !errors.Is(err, ErrRawAPIKeyNotAllowed) { - t.Fatalf("expected ErrRawAPIKeyNotAllowed, got %v", err) - } - }) - - t.Run("ignores raw api_key in non-selected profiles", func(t *testing.T) { - repo := NewFSRepository(fstest.MapFS{ - "profiles/raw.yaml": profileMapFile(` -id: raw-profile -endpoint: http://localhost:8000/v1 -model: model -api_key: secret -`), - "profiles/valid.yaml": profileMapFile(` -id: valid-profile -endpoint: http://localhost:8000/v1 -model: model -`), - }, "profiles") - - p, err := repo.GetProfile(ctx, "valid-profile") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if p.ID != "valid-profile" { - t.Fatalf("unexpected profile: %+v", p) - } - }) - - t.Run("rejects duplicate IDs within one source", func(t *testing.T) { - repo := NewFSRepository(fstest.MapFS{ - "profiles/a.yaml": profileMapFile(` -id: duplicate-profile -endpoint: http://localhost:8000/v1 -model: first -`), - "profiles/nested/b.yaml": profileMapFile(` -id: duplicate-profile -endpoint: http://localhost:8000/v1 -model: second -`), - }, "profiles") - - _, err := repo.GetProfile(ctx, "duplicate-profile") - if !errors.Is(err, ErrInvalidProfile) { - t.Fatalf("expected ErrInvalidProfile, got %v", err) - } - for _, want := range []string{"duplicate execution profile id", "a.yaml", "nested/b.yaml"} { - if !strings.Contains(err.Error(), want) { - t.Fatalf("expected error to contain %q, got %v", want, err) - } - } - }) -} - -func TestProfileRepositoriesRejectInvalidEndpoints(t *testing.T) { - tests := []struct { - name string - endpoint string - withBackend bool - }{ - {name: "relative", endpoint: "/v1"}, - {name: "missing host", endpoint: "https:///v1"}, - {name: "unsupported scheme", endpoint: "ftp://provider.example/v1"}, - {name: "user information", endpoint: "https://user@provider.example/v1"}, - {name: "query", endpoint: "https://provider.example/v1?mode=chat"}, - {name: "fragment", endpoint: "https://provider.example/v1#chat"}, - {name: "backend with invalid override", endpoint: "/v1", withBackend: true}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - backend := "" - if tc.withBackend { - backend = "backend: openrouter\n" - } - repo := NewFSRepository(fstest.MapFS{ - "profiles/invalid.yaml": profileMapFile(fmt.Sprintf( - "id: invalid-endpoint\nmodel: model\n%sendpoint: %q\n", - backend, - tc.endpoint, - )), - }, "profiles") - - _, err := repo.GetProfile(context.Background(), "invalid-endpoint") - if !errors.Is(err, ErrInvalidProfile) { - t.Fatalf("expected ErrInvalidProfile, got %v", err) - } - }) - } -} - -func TestProfileRepositoriesValidateExtraParams(t *testing.T) { - const validProfile = ` -id: selected-profile -endpoint: http://localhost:8000/v1 -model: model -extra_params: - string_value: enabled - object_value: - nested: true - array_value: - - first - - 3 -` - tests := []struct { - name string - definition string - wantErr bool - diagnostics []string - }{ - {name: "valid nested values", definition: validProfile}, - { - name: "empty key", - definition: ` -id: selected-profile -endpoint: http://localhost:8000/v1 -model: model -extra_params: - "": value -`, - wantErr: true, - diagnostics: []string{"extra_params", "key must not be empty"}, - }, - { - name: "non-finite value", - definition: ` -id: selected-profile -endpoint: http://localhost:8000/v1 -model: model -extra_params: - invalid: .nan -`, - wantErr: true, - diagnostics: []string{"extra_params.invalid", "must be finite"}, - }, - { - name: "nested non-finite value", - definition: ` -id: selected-profile -endpoint: http://localhost:8000/v1 -model: model -extra_params: - outer: - invalid: .inf -`, - wantErr: true, - diagnostics: []string{"extra_params.outer.invalid", "must be finite"}, - }, - { - name: "unsupported decoded value", - definition: ` -id: selected-profile -endpoint: http://localhost:8000/v1 -model: model -extra_params: - timestamp: 2026-08-11T12:34:56Z -`, - wantErr: true, - diagnostics: []string{"extra_params.timestamp", "unsupported JSON value type"}, - }, - { - name: "excessive nesting", - definition: deeplyNestedExtraParamsProfile(101), - wantErr: true, - diagnostics: []string{"extra_params", "JSON container depth limit exceeded"}, - }, - } - - for _, source := range profileRepositorySources() { - for _, tc := range tests { - t.Run(source.name+"/"+tc.name, func(t *testing.T) { - repo := source.newRepository(t, map[string]string{"selected.yaml": tc.definition}) - got, err := repo.GetProfile(context.Background(), "selected-profile") - if tc.wantErr { - if !errors.Is(err, ErrInvalidProfile) { - t.Fatalf("expected ErrInvalidProfile, got %v", err) - } - if !strings.Contains(err.Error(), "selected.yaml") { - t.Fatalf("expected source path in error, got %v", err) - } - for _, diagnostic := range tc.diagnostics { - if !strings.Contains(err.Error(), diagnostic) { - t.Fatalf("expected error to contain %q, got %v", diagnostic, err) - } - } - return - } - if err != nil { - t.Fatalf("load valid profile: %v", err) - } - if got.ExtraParams["string_value"] != "enabled" { - t.Fatalf("unexpected copied extra params: %#v", got.ExtraParams) - } - objectValue, objectOK := got.ExtraParams["object_value"].(map[string]any) - arrayValue, arrayOK := got.ExtraParams["array_value"].([]any) - if !objectOK || objectValue["nested"] != true || - !arrayOK || len(arrayValue) != 2 || arrayValue[0] != "first" || arrayValue[1] != 3 { - t.Fatalf("unexpected copied nested extra params: %#v", got.ExtraParams) - } - }) - } - } -} - -func TestProfileRepositoriesSelectCanonicalYAMLID(t *testing.T) { - const validProfile = ` -id: selected-profile -endpoint: http://localhost:8000/v1 -model: selected-model -` - tests := []struct { - name string - files map[string]string - wantErr error - diagnostics []string - }{ - { - name: "same stem unknown field with different id is unrelated", - files: map[string]string{ - "selected-profile.yaml": ` -id: unrelated-profile -endpoint: http://localhost:8000/v1 -model: unrelated -unknown: true -`, - "valid.yaml": validProfile, - }, - }, - { - name: "same stem unidentifiable yaml is unrelated", - files: map[string]string{ - "selected-profile.yaml": "id: [", - "valid.yaml": validProfile, - }, - }, - { - name: "same stem raw key with different id is unrelated", - files: map[string]string{ - "selected-profile.yaml": ` -id: unrelated-profile -endpoint: http://localhost:8000/v1 -model: unrelated -api_key: secret -`, - "valid.yaml": validProfile, - }, - }, - { - name: "leading and trailing whitespace is normalized", - files: map[string]string{ - "padded.yaml": ` -id: " selected-profile " -endpoint: http://localhost:8000/v1 -model: selected-model -`, - }, - }, - { - name: "blank id is unrelated", - files: map[string]string{ - "selected-profile.yaml": ` -id: " " -endpoint: http://localhost:8000/v1 -model: unrelated -`, - }, - wantErr: ErrProfileNotFound, - }, - { - name: "normalized duplicates are ambiguous", - files: map[string]string{ - "first.yaml": validProfile, - "nested/second.yaml": ` -id: " selected-profile " -endpoint: http://localhost:8000/v1 -model: duplicate -`, - }, - wantErr: ErrInvalidProfile, - diagnostics: []string{"duplicate execution profile id", "first.yaml", "nested/second.yaml"}, - }, - { - name: "selected unknown field is authoritative", - files: map[string]string{ - "malformed.yaml": ` -id: selected-profile -endpoint: http://localhost:8000/v1 -model: selected-model -unknown: true -`, - }, - wantErr: ErrInvalidYAML, - diagnostics: []string{"malformed.yaml"}, - }, - { - name: "selected raw key is authoritative", - files: map[string]string{ - "insecure.yaml": ` -id: selected-profile -endpoint: http://localhost:8000/v1 -model: selected-model -api_key: secret -`, - }, - wantErr: ErrRawAPIKeyNotAllowed, - diagnostics: []string{"insecure.yaml"}, - }, - { - name: "selected identity in an additional document is authoritative", - files: map[string]string{ - "additional-document.yaml": ` ---- ---- -id: selected-profile -endpoint: http://localhost:8000/v1 -model: selected-model -`, - }, - wantErr: ErrInvalidYAML, - diagnostics: []string{"additional-document.yaml", "exactly one YAML document"}, - }, - } - - for _, source := range profileRepositorySources() { - for _, tc := range tests { - t.Run(source.name+"/"+tc.name, func(t *testing.T) { - repo := source.newRepository(t, tc.files) - got, err := repo.GetProfile(context.Background(), " selected-profile ") - if tc.wantErr != nil { - if !errors.Is(err, tc.wantErr) { - t.Fatalf("expected %v, got %v", tc.wantErr, err) - } - for _, diagnostic := range tc.diagnostics { - if !strings.Contains(err.Error(), diagnostic) { - t.Fatalf("expected error to contain %q, got %v", diagnostic, err) - } - } - return - } - if err != nil { - t.Fatalf("load selected profile: %v", err) - } - if got.ID != "selected-profile" || got.Model != "selected-model" { - t.Fatalf("unexpected selected profile: %+v", got) - } - }) - } - } -} - -func TestProfileRepositoriesRequireOneYAMLDocument(t *testing.T) { - const profile = ` -id: selected-profile -endpoint: http://localhost:8000/v1 -model: selected-model -` - tests := []struct { - name string - suffix string - wantErr bool - }{ - {name: "comments and trailing whitespace", suffix: "\n# trailing comment\n\n"}, - {name: "second populated document", suffix: "\n---\nid: another\n", wantErr: true}, - {name: "second empty document", suffix: "\n---\n", wantErr: true}, - {name: "malformed trailing yaml", suffix: "\n---\n[", wantErr: true}, - {name: "raw key in trailing document", suffix: "\n---\napi_key: secret\n", wantErr: true}, - } - - for _, source := range profileRepositorySources() { - for _, tc := range tests { - t.Run(source.name+"/"+tc.name, func(t *testing.T) { - repo := source.newRepository(t, map[string]string{"definition.yaml": profile + tc.suffix}) - got, err := repo.GetProfile(context.Background(), "selected-profile") - if tc.wantErr { - if !errors.Is(err, ErrInvalidYAML) { - t.Fatalf("expected ErrInvalidYAML, got %v", err) - } - if !strings.Contains(err.Error(), "definition.yaml") { - t.Fatalf("expected source path in error, got %v", err) - } - return - } - if err != nil { - t.Fatalf("load one-document profile: %v", err) - } - if got.ID != "selected-profile" { - t.Fatalf("unexpected profile: %+v", got) - } - }) - } - } -} - -func TestProfileRepositoriesPreserveOverlayFallbackRules(t *testing.T) { - fallback := staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "selected-profile": {ID: "selected-profile", Endpoint: "http://fallback", Model: "fallback-model"}, - }} - tests := []struct { - name string - files map[string]string - wantModel string - wantErr error - }{ - { - name: "same stem malformed different id falls back", - files: map[string]string{ - "selected-profile.yaml": ` -id: unrelated-profile -endpoint: http://localhost:8000/v1 -model: unrelated -unknown: true -`, - }, - wantModel: "fallback-model", - }, - { - name: "blank id falls back", - files: map[string]string{ - "selected-profile.yaml": ` -id: " " -endpoint: http://localhost:8000/v1 -model: unrelated -`, - }, - wantModel: "fallback-model", - }, - { - name: "selected malformed profile stops fallback", - files: map[string]string{ - "other-name.yaml": ` -id: selected-profile -endpoint: http://localhost:8000/v1 -model: selected -unknown: true -`, - }, - wantErr: ErrInvalidYAML, - }, - } - - for _, source := range profileRepositorySources() { - for _, tc := range tests { - t.Run(source.name+"/"+tc.name, func(t *testing.T) { - primary := source.newRepository(t, tc.files) - got, err := NewOverlayRepository(primary, fallback).GetProfile(context.Background(), "selected-profile") - if tc.wantErr != nil { - if !errors.Is(err, tc.wantErr) { - t.Fatalf("expected %v, got %v", tc.wantErr, err) - } - return - } - if err != nil { - t.Fatalf("load fallback profile: %v", err) - } - if got.Model != tc.wantModel { - t.Fatalf("model = %q, want %q", got.Model, tc.wantModel) - } - }) - } - } -} - -func TestProfileRepositoryReadsSourcesFreshOnEveryLookup(t *testing.T) { - newSource := func() (*recordingProfileFS, Repository) { - fsys := &recordingProfileFS{FS: fstest.MapFS{ - "target.yaml": profileMapFile(` -id: target -endpoint: http://localhost:8000/v1 -model: target-model -`), - "unrelated.yaml": profileMapFile(` -id: unrelated -endpoint: http://localhost:8000/v1 -model: unrelated-model -`), - }} - return fsys, NewFSRepository(fsys, ".") - } - - t.Run("selected source", func(t *testing.T) { - fsys, repo := newSource() - for lookup := 1; lookup <= 2; lookup++ { - got, err := repo.GetProfile(context.Background(), "target") - if err != nil { - t.Fatalf("lookup %d: %v", lookup, err) - } - if got.Model != "target-model" { - t.Fatalf("lookup %d model = %q", lookup, got.Model) - } - for _, name := range []string{"target.yaml", "unrelated.yaml"} { - if count := fsys.openCount(name); count != lookup { - t.Fatalf("%s opens after lookup %d = %d, want %d", name, lookup, count, lookup) - } - } - } - }) - - t.Run("overlay fallthrough", func(t *testing.T) { - primaryFS := &recordingProfileFS{FS: fstest.MapFS{ - "unrelated.yaml": profileMapFile(` -id: unrelated -endpoint: http://localhost:8000/v1 -model: unrelated-model -`), - }} - fallbackFS, fallback := newSource() - repo := NewOverlayRepository(NewFSRepository(primaryFS, "."), fallback) - - for lookup := 1; lookup <= 2; lookup++ { - got, err := repo.GetProfile(context.Background(), "target") - if err != nil { - t.Fatalf("lookup %d: %v", lookup, err) - } - if got.Model != "target-model" { - t.Fatalf("lookup %d model = %q", lookup, got.Model) - } - if count := primaryFS.openCount("unrelated.yaml"); count != lookup { - t.Fatalf("primary opens after lookup %d = %d, want %d", lookup, count, lookup) - } - if count := fallbackFS.openCount("target.yaml"); count != lookup { - t.Fatalf("fallback opens after lookup %d = %d, want %d", lookup, count, lookup) - } - } - }) -} - -func TestProfileRepositoriesRejectInvalidExecutionSettings(t *testing.T) { - ctx := context.Background() - - t.Run("operating-system filesystem", func(t *testing.T) { - dir := t.TempDir() - writeProfileTestFile(t, filepath.Join(dir, "invalid.yaml"), ` -id: invalid -endpoint: http://localhost:8000/v1 -model: model -temperature: .nan -`) - - _, err := NewFilesystemRepository(dir).GetProfile(ctx, "invalid") - if !errors.Is(err, ErrInvalidProfile) { - t.Fatalf("expected ErrInvalidProfile, got %v", err) - } - }) - - t.Run("fs.FS", func(t *testing.T) { - repo := NewFSRepository(fstest.MapFS{ - "profiles/invalid.yaml": profileMapFile(` -id: invalid -endpoint: http://localhost:8000/v1 -model: model -top_p: .inf -`), - }, "profiles") - - _, err := repo.GetProfile(ctx, "invalid") - if !errors.Is(err, ErrInvalidProfile) { - t.Fatalf("expected ErrInvalidProfile, got %v", err) - } - }) -} - -func TestProfileRepositoriesValidateDerivedDefinitions(t *testing.T) { - tests := []struct { - name string - files map[string]string - wantError error - wantBaseID string - wantProfile bool - }{ - { - name: "alias is locally valid and normalizes base id", - files: map[string]string{"alias.yaml": ` -id: selected-profile -base_profile: " base-profile " -`}, - wantBaseID: "base-profile", - wantProfile: true, - }, - { - name: "derived endpoint remains valid", - files: map[string]string{"invalid.yaml": ` -id: selected-profile -base_profile: base-profile -endpoint: /v1 -`}, - wantError: ErrInvalidProfile, - }, - { - name: "derived settings remain valid", - files: map[string]string{"invalid.yaml": ` -id: selected-profile -base_profile: base-profile -top_p: 1.1 -`}, - wantError: ErrInvalidProfile, - }, - { - name: "derived extra params remain valid", - files: map[string]string{"invalid.yaml": ` -id: selected-profile -base_profile: base-profile -extra_params: - timestamp: 2026-08-11T12:34:56Z -`}, - wantError: ErrInvalidProfile, - }, - { - name: "derived raw key remains prohibited", - files: map[string]string{"invalid.yaml": ` -id: selected-profile -base_profile: base-profile -api_key: secret -`}, - wantError: ErrRawAPIKeyNotAllowed, - }, - { - name: "derived duplicate id remains invalid", - files: map[string]string{ - "first.yaml": "id: selected-profile\nbase_profile: first-base\n", - "second.yaml": "id: selected-profile\nbase_profile: second-base\n", - }, - wantError: ErrInvalidProfile, - }, - { - name: "derived extra document remains invalid", - files: map[string]string{"invalid.yaml": ` -id: selected-profile -base_profile: base-profile ---- -id: other -`}, - wantError: ErrInvalidYAML, - }, - { - name: "standalone profile remains complete", - files: map[string]string{"invalid.yaml": "id: selected-profile\n"}, - wantError: ErrInvalidProfile, - }, - } - - for _, source := range profileRepositorySources() { - for _, tc := range tests { - t.Run(source.name+"/"+tc.name, func(t *testing.T) { - repo := source.newRepository(t, tc.files) - got, err := repo.GetProfile(context.Background(), "selected-profile") - if tc.wantError != nil { - if !errors.Is(err, tc.wantError) { - t.Fatalf("error = %v, want %v", err, tc.wantError) - } - return - } - if err != nil || !tc.wantProfile { - t.Fatalf("profile = %+v, error = %v, want valid derived definition", got, err) - } - if got.BaseProfileID != tc.wantBaseID { - t.Fatalf("BaseProfileID = %q, want %q", got.BaseProfileID, tc.wantBaseID) - } - }) - } - } -} - -func TestOverlayRepository(t *testing.T) { - ctx := context.Background() - primaryProfile := &domain.ExecutionProfile{ID: "shared", Endpoint: "http://primary", Model: "primary"} - fallbackProfile := &domain.ExecutionProfile{ID: "shared", Endpoint: "http://fallback", Model: "fallback"} - - t.Run("returns primary matches before fallback matches", func(t *testing.T) { - repo := NewOverlayRepository( - staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": primaryProfile}}, - staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}}, - ) - - p, err := repo.GetProfile(ctx, "shared") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if p.Model != "primary" { - t.Fatalf("expected primary profile, got %+v", p) - } - }) - - t.Run("falls back on primary not found", func(t *testing.T) { - repo := NewOverlayRepository( - staticProfileRepo{}, - staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}}, - ) - - p, err := repo.GetProfile(ctx, "shared") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if p.Model != "fallback" { - t.Fatalf("expected fallback profile, got %+v", p) - } - }) - - t.Run("does not fall back after primary load errors", func(t *testing.T) { - for _, tc := range []struct { - name string - err error - }{ - {name: "invalid yaml", err: ErrInvalidYAML}, - {name: "invalid profile", err: ErrInvalidProfile}, - {name: "raw api key", err: ErrRawAPIKeyNotAllowed}, - } { - t.Run(tc.name, func(t *testing.T) { - repo := NewOverlayRepository( - staticProfileRepo{err: tc.err}, - staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}}, - ) - - _, err := repo.GetProfile(ctx, "shared") - if !errors.Is(err, tc.err) { - t.Fatalf("expected %v, got %v", tc.err, err) - } - }) - } - }) - - t.Run("returns not found when both sources miss", func(t *testing.T) { - repo := NewOverlayRepository(staticProfileRepo{}, staticProfileRepo{}) - - _, err := repo.GetProfile(ctx, "missing") - if !errors.Is(err, ErrProfileNotFound) { - t.Fatalf("expected ErrProfileNotFound, got %v", err) - } - }) - - t.Run("nil primary uses fallback", func(t *testing.T) { - repo := NewOverlayRepository(nil, staticProfileRepo{profiles: map[string]*domain.ExecutionProfile{"shared": fallbackProfile}}) - - p, err := repo.GetProfile(ctx, "shared") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if p.Model != "fallback" { - t.Fatalf("expected fallback profile, got %+v", p) - } - }) - - t.Run("nil fallback returns not found after primary miss", func(t *testing.T) { - repo := NewOverlayRepository(staticProfileRepo{}, nil) - - _, err := repo.GetProfile(ctx, "missing") - if !errors.Is(err, ErrProfileNotFound) { - t.Fatalf("expected ErrProfileNotFound, got %v", err) - } - }) -} - -type profileRepositorySource struct { - name string - newRepository func(t *testing.T, files map[string]string) Repository -} - -type recordingProfileFS struct { - fs.FS - mu sync.Mutex - opened []string -} - -func (f *recordingProfileFS) Open(name string) (fs.File, error) { - f.mu.Lock() - f.opened = append(f.opened, name) - f.mu.Unlock() - return f.FS.Open(name) -} - -func (f *recordingProfileFS) openCount(name string) int { - f.mu.Lock() - defer f.mu.Unlock() - count := 0 - for _, opened := range f.opened { - if opened == name { - count++ - } - } - return count -} - -func profileRepositorySources() []profileRepositorySource { - return []profileRepositorySource{ - { - name: "operating system", - newRepository: func(t *testing.T, files map[string]string) Repository { - t.Helper() - root := t.TempDir() - for name, content := range files { - filePath := filepath.Join(root, filepath.FromSlash(name)) - if err := os.MkdirAll(filepath.Dir(filePath), 0o755); err != nil { - t.Fatalf("create profile directory: %v", err) - } - writeProfileTestFile(t, filePath, content) - } - return NewFilesystemRepository(root) - }, - }, - { - name: "filesystem", - newRepository: func(t *testing.T, files map[string]string) Repository { - t.Helper() - fsys := make(fstest.MapFS, len(files)) - for name, content := range files { - fsys[name] = profileMapFile(content) - } - return NewFSRepository(fsys, ".") - }, - }, - } -} - -func deeplyNestedExtraParamsProfile(depth int) string { - var definition strings.Builder - definition.WriteString("id: selected-profile\nendpoint: http://localhost:8000/v1\nmodel: model\nextra_params:\n") - for level := 0; level < depth; level++ { - fmt.Fprintf(&definition, "%slevel_%d:\n", strings.Repeat(" ", level+1), level) - } - fmt.Fprintf(&definition, "%svalue: true\n", strings.Repeat(" ", depth+1)) - return definition.String() -} - -func profileMapFile(content string) *fstest.MapFile { - return &fstest.MapFile{Data: []byte(strings.TrimLeft(content, "\n"))} -} - -type staticProfileRepo struct { - profiles map[string]*domain.ExecutionProfile - err error -} - -func (r staticProfileRepo) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) { - if r.err != nil { - return nil, r.err - } - if p, ok := r.profiles[id]; ok { - cp := *p - return &cp, nil - } - return nil, ErrProfileNotFound -} diff --git a/internal/profile/resolving_repository.go b/internal/profile/resolving_repository.go deleted file mode 100644 index 63ae075..0000000 --- a/internal/profile/resolving_repository.go +++ /dev/null @@ -1,160 +0,0 @@ -package profile - -import ( - "context" - "errors" - "fmt" - "strings" - - "gitea.maximumdirect.net/eric/promptkit/internal/domain" - "gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue" -) - -const maximumProfileChainLength = 32 - -type resolvingRepository struct { - source Repository -} - -// NewResolvingRepository resolves inherited profile definitions from source. -func NewResolvingRepository(source Repository) Repository { - return &resolvingRepository{source: source} -} - -func (r *resolvingRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) { - if r == nil || r.source == nil { - return nil, fmt.Errorf("%w: profile repository is required", ErrInvalidProfile) - } - requestedID := strings.TrimSpace(id) - if requestedID == "" { - return nil, fmt.Errorf("%w: profile id is required", ErrInvalidProfile) - } - if err := ctx.Err(); err != nil { - return nil, err - } - - profile, err := r.getRawProfile(ctx, requestedID) - if err != nil { - return nil, err - } - if profile == nil { - return nil, fmt.Errorf("%w: selected profile %q is nil", ErrInvalidProfile, requestedID) - } - - chain := []*domain.ExecutionProfile{profile} - chainIDs := []string{requestedID} - visited := map[string]struct{}{requestedID: {}} - current := profile - - for { - baseID := strings.TrimSpace(current.BaseProfileID) - if baseID == "" { - break - } - if err := ctx.Err(); err != nil { - return nil, err - } - if _, seen := visited[baseID]; seen { - return nil, fmt.Errorf("%w: profile inheritance cycle %s", ErrInvalidProfile, joinProfileChain(chainIDs, baseID)) - } - if len(chain) >= maximumProfileChainLength { - return nil, fmt.Errorf("%w: profile inheritance chain exceeds %d profiles: %s", ErrInvalidProfile, maximumProfileChainLength, joinProfileChain(chainIDs, baseID)) - } - - base, err := r.getRawProfile(ctx, baseID) - if err != nil { - if errors.Is(err, ErrProfileNotFound) { - return nil, fmt.Errorf("%w: base profile %q is missing in chain %s", ErrInvalidProfile, baseID, joinProfileChain(chainIDs, baseID)) - } - return nil, fmt.Errorf("%w: failed to load base profile %q in chain %s: %w", ErrInvalidProfile, baseID, joinProfileChain(chainIDs, baseID), err) - } - if base == nil { - return nil, fmt.Errorf("%w: base profile %q is nil in chain %s", ErrInvalidProfile, baseID, joinProfileChain(chainIDs, baseID)) - } - - chain = append(chain, base) - chainIDs = append(chainIDs, baseID) - visited[baseID] = struct{}{} - current = base - } - - resolved, err := mergeProfileChain(chain) - if err != nil { - return nil, fmt.Errorf("%w: resolved profile chain %s: %w", ErrInvalidProfile, strings.Join(chainIDs, " -> "), err) - } - if err := validateResolvedProfile(resolved); err != nil { - return nil, fmt.Errorf("%w: resolved profile chain %s: %w", ErrInvalidProfile, strings.Join(chainIDs, " -> "), err) - } - return resolved, nil -} - -func (r *resolvingRepository) getRawProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) { - profile, err := r.source.GetProfile(ctx, id) - if err != nil { - return nil, err - } - if err := ctx.Err(); err != nil { - return nil, err - } - return profile, nil -} - -func joinProfileChain(chain []string, next string) string { - return strings.Join(append(append([]string(nil), chain...), next), " -> ") -} - -func mergeProfileChain(chain []*domain.ExecutionProfile) (*domain.ExecutionProfile, error) { - resolved := &domain.ExecutionProfile{ID: chain[0].ID} - for index := len(chain) - 1; index >= 0; index-- { - definition := chain[index] - if strings.TrimSpace(definition.BackendID) != "" { - resolved.BackendID = definition.BackendID - } - if strings.TrimSpace(definition.Endpoint) != "" { - resolved.Endpoint = definition.Endpoint - } - if strings.TrimSpace(definition.Model) != "" { - resolved.Model = definition.Model - } - if definition.Temperature != 0 { - resolved.Temperature = definition.Temperature - } - if definition.MaxTokens != 0 { - resolved.MaxTokens = definition.MaxTokens - } - if definition.TopP != 0 { - resolved.TopP = definition.TopP - } - if definition.TimeoutSeconds != 0 { - resolved.TimeoutSeconds = definition.TimeoutSeconds - } - if strings.TrimSpace(definition.ServiceTier) != "" { - resolved.ServiceTier = definition.ServiceTier - } - if strings.TrimSpace(definition.ReasoningEffort) != "" { - resolved.ReasoningEffort = definition.ReasoningEffort - } - if strings.TrimSpace(definition.APIKeyEnv) != "" { - resolved.APIKeyEnv = definition.APIKeyEnv - } - resolved.APIKeyRequired = resolved.APIKeyRequired || definition.APIKeyRequired - if len(definition.ExtraParams) != 0 { - extraParams, err := jsonvalue.CopyMap(definition.ExtraParams) - if err != nil { - return nil, err - } - resolved.ExtraParams = extraParams - } - } - resolved.ID = chain[0].ID - resolved.BaseProfileID = "" - return resolved, nil -} - -func validateResolvedProfile(profile *domain.ExecutionProfile) error { - if profile == nil { - return errors.New("resolved profile is required") - } - profile.BaseProfileID = "" - return NormalizeAndValidateDefinition(profile) -} diff --git a/internal/profile/resolving_repository_test.go b/internal/profile/resolving_repository_test.go deleted file mode 100644 index 1ca6079..0000000 --- a/internal/profile/resolving_repository_test.go +++ /dev/null @@ -1,418 +0,0 @@ -package profile - -import ( - "context" - "errors" - "fmt" - "io/fs" - "reflect" - "strings" - "sync" - "testing" - "testing/fstest" - - "gitea.maximumdirect.net/eric/promptkit/internal/domain" -) - -func TestResolvingRepositoryMergesProfileChain(t *testing.T) { - repo := &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{ - "leaf": { - ID: "leaf", - BaseProfileID: "middle", - BackendID: "leaf-backend", - TopP: 0.8, - TimeoutSeconds: 45, - ReasoningEffort: "high", - }, - "middle": { - ID: "middle", - BaseProfileID: "root", - Endpoint: "https://middle.example/v1", - Model: "middle-model", - MaxTokens: 256, - APIKeyEnv: "MIDDLE_API_KEY", - APIKeyRequired: true, - ExtraParams: map[string]any{"middle": map[string]any{"value": "middle"}}, - }, - "root": { - ID: "root", - BackendID: "root-backend", - Endpoint: "https://root.example/v1", - Model: "root-model", - Temperature: 0.3, - ServiceTier: "priority", - ExtraParams: map[string]any{"root": "value"}, - }, - }} - - got, err := NewResolvingRepository(repo).GetProfile(context.Background(), "leaf") - if err != nil { - t.Fatalf("resolve profile: %v", err) - } - want := &domain.ExecutionProfile{ - ID: "leaf", - BackendID: "leaf-backend", - Endpoint: "https://middle.example/v1", - Model: "middle-model", - Temperature: 0.3, - MaxTokens: 256, - TopP: 0.8, - TimeoutSeconds: 45, - ServiceTier: "priority", - ReasoningEffort: "high", - APIKeyEnv: "MIDDLE_API_KEY", - APIKeyRequired: true, - ExtraParams: map[string]any{"middle": map[string]any{"value": "middle"}}, - } - if !reflect.DeepEqual(got, want) { - t.Fatalf("resolved profile:\n got %#v\nwant %#v", got, want) - } -} - -func TestResolvingRepositoryRejectsMissingSourceAndProfileID(t *testing.T) { - if _, err := NewResolvingRepository(nil).GetProfile(context.Background(), "profile"); !errors.Is(err, ErrInvalidProfile) { - t.Fatalf("nil source error = %v, want ErrInvalidProfile", err) - } - - repo := &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{}} - if _, err := NewResolvingRepository(repo).GetProfile(context.Background(), " \t "); !errors.Is(err, ErrInvalidProfile) { - t.Fatalf("blank id error = %v, want ErrInvalidProfile", err) - } - if got := repo.callCount(" "); got != 0 { - t.Fatalf("blank id looked up source %d times", got) - } -} - -func TestResolvingRepositoryCopiesExtraParams(t *testing.T) { - baseParams := map[string]any{"nested": map[string]any{"value": "base"}} - repo := &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{ - "child": {ID: "child", BaseProfileID: "base"}, - "base": { - ID: "base", - Endpoint: "https://base.example/v1", - Model: "model", - ExtraParams: baseParams, - }, - }} - resolver := NewResolvingRepository(repo) - - first, err := resolver.GetProfile(context.Background(), "child") - if err != nil { - t.Fatalf("resolve inherited map: %v", err) - } - first.ExtraParams["nested"].(map[string]any)["value"] = "mutated" - second, err := resolver.GetProfile(context.Background(), "child") - if err != nil { - t.Fatalf("resolve inherited map again: %v", err) - } - if got := second.ExtraParams["nested"].(map[string]any)["value"]; got != "base" { - t.Fatalf("later result retained mutation: %v", got) - } - if got := baseParams["nested"].(map[string]any)["value"]; got != "base" { - t.Fatalf("source map retained mutation: %v", got) - } - - repo.set("child", &domain.ExecutionProfile{ - ID: "child", - BaseProfileID: "base", - ExtraParams: map[string]any{"child": "replacement"}, - }) - replaced, err := resolver.GetProfile(context.Background(), "child") - if err != nil { - t.Fatalf("resolve replacement map: %v", err) - } - if !reflect.DeepEqual(replaced.ExtraParams, map[string]any{"child": "replacement"}) { - t.Fatalf("extra params = %#v, want complete child replacement", replaced.ExtraParams) - } -} - -func TestResolvingRepositoryUsesRawOverlayForEachLookup(t *testing.T) { - leafSource := NewFSRepository(profileTestFS(map[string]string{ - "leaf.yaml": "id: leaf\nbase_profile: base\n", - }), ".") - fallback := NewFSRepository(profileTestFS(map[string]string{ - "base.yaml": "id: base\nendpoint: https://fallback.example/v1\nmodel: fallback-model\n", - }), ".") - overlay := NewOverlayRepository(leafSource, fallback) - resolver := NewResolvingRepository(overlay) - - got, err := resolver.GetProfile(context.Background(), "leaf") - if err != nil { - t.Fatalf("resolve fallback base: %v", err) - } - if got.Model != "fallback-model" { - t.Fatalf("fallback base model = %q", got.Model) - } - - shadowing := NewOverlayRepository(NewFSRepository(profileTestFS(map[string]string{ - "leaf.yaml": "id: leaf\nbase_profile: base\n", - "base.yaml": "id: base\nendpoint: https://primary.example/v1\nmodel: primary-model\n", - }), "."), fallback) - got, err = NewResolvingRepository(shadowing).GetProfile(context.Background(), "leaf") - if err != nil { - t.Fatalf("resolve shadowed base: %v", err) - } - if got.Model != "primary-model" || got.Endpoint != "https://primary.example/v1" { - t.Fatalf("shadowed base = %+v", got) - } -} - -func TestResolvingRepositoryReportsSafetyAndSourceErrors(t *testing.T) { - sourceErr := errors.New("source failure") - tests := []struct { - name string - repo *resolvingTestRepository - id string - want []error - wantNot error - contains []string - }{ - { - name: "missing selected profile preserves not found", - repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{}}, - id: "missing", - want: []error{ErrProfileNotFound}, - wantNot: ErrInvalidProfile, - }, - { - name: "missing base is invalid but not not found", - repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{ - "leaf": {ID: "leaf", BaseProfileID: "missing"}, - }}, - id: "leaf", - want: []error{ErrInvalidProfile}, - wantNot: ErrProfileNotFound, - contains: []string{"missing", "leaf -> missing"}, - }, - { - name: "direct cycle", - repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{ - "a": {ID: "a", BaseProfileID: "a"}, - }}, - id: "a", - want: []error{ErrInvalidProfile}, - contains: []string{"a -> a"}, - }, - { - name: "indirect cycle", - repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{ - "a": {ID: "a", BaseProfileID: "b"}, - "b": {ID: "b", BaseProfileID: "c"}, - "c": {ID: "c", BaseProfileID: "a"}, - }}, - id: "a", - want: []error{ErrInvalidProfile}, - contains: []string{"a -> b -> c -> a"}, - }, - { - name: "nil result", - repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{ - "leaf": nil, - }}, - id: "leaf", - want: []error{ErrInvalidProfile}, - }, - { - name: "incomplete resolved profile", - repo: &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{ - "leaf": {ID: "leaf", BaseProfileID: "base"}, - "base": {ID: "base", Model: "model"}, - }}, - id: "leaf", - want: []error{ErrInvalidProfile}, - }, - { - name: "base source error is retained", - repo: &resolvingTestRepository{ - profiles: map[string]*domain.ExecutionProfile{"leaf": {ID: "leaf", BaseProfileID: "base"}}, - errors: map[string]error{"base": sourceErr}, - }, - id: "leaf", - want: []error{ErrInvalidProfile, sourceErr}, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - _, err := NewResolvingRepository(tc.repo).GetProfile(context.Background(), tc.id) - for _, want := range tc.want { - if !errors.Is(err, want) { - t.Fatalf("error = %v, want %v", err, want) - } - } - if tc.wantNot != nil && errors.Is(err, tc.wantNot) { - t.Fatalf("error = %v, must not match %v", err, tc.wantNot) - } - for _, fragment := range tc.contains { - if !strings.Contains(err.Error(), fragment) { - t.Fatalf("error = %v, want %q", err, fragment) - } - } - }) - } -} - -func TestResolvingRepositoryEnforcesChainLength(t *testing.T) { - for _, count := range []int{maximumProfileChainLength, maximumProfileChainLength + 1} { - t.Run(fmt.Sprintf("%d profiles", count), func(t *testing.T) { - profiles := make(map[string]*domain.ExecutionProfile, count) - for index := 1; index <= count; index++ { - id := fmt.Sprintf("profile-%d", index) - definition := &domain.ExecutionProfile{ID: id} - if index == count { - definition.Endpoint = "https://root.example/v1" - definition.Model = "model" - } else { - definition.BaseProfileID = fmt.Sprintf("profile-%d", index+1) - } - profiles[id] = definition - } - - got, err := NewResolvingRepository(&resolvingTestRepository{profiles: profiles}).GetProfile(context.Background(), "profile-1") - if count == maximumProfileChainLength { - if err != nil || got == nil { - t.Fatalf("profile = %+v, error = %v, want accepted chain", got, err) - } - return - } - if !errors.Is(err, ErrInvalidProfile) { - t.Fatalf("error = %v, want ErrInvalidProfile", err) - } - }) - } -} - -func TestResolvingRepositoryIsFreshAndCancellationAware(t *testing.T) { - repo := &resolvingTestRepository{profiles: map[string]*domain.ExecutionProfile{ - "leaf": {ID: "leaf", BaseProfileID: "base"}, - "base": {ID: "base", Endpoint: "https://base.example/v1", Model: "first", ExtraParams: map[string]any{"nested": map[string]any{"value": "first"}}}, - }} - resolver := NewResolvingRepository(repo) - - first, err := resolver.GetProfile(context.Background(), "leaf") - if err != nil || first.Model != "first" { - t.Fatalf("first result=(%+v, %v)", first, err) - } - repo.set("base", &domain.ExecutionProfile{ID: "base", Endpoint: "https://base.example/v1", Model: "second", ExtraParams: map[string]any{"nested": map[string]any{"value": "second"}}}) - second, err := resolver.GetProfile(context.Background(), "leaf") - if err != nil || second.Model != "second" { - t.Fatalf("second result=(%+v, %v)", second, err) - } - - canceled, cancel := context.WithCancel(context.Background()) - cancel() - if _, err := resolver.GetProfile(canceled, "leaf"); !errors.Is(err, context.Canceled) { - t.Fatalf("canceled lookup error = %v", err) - } - if got := repo.callCount("leaf"); got != 2 { - t.Fatalf("calls after canceled lookup = %d, want 2", got) - } - - duringTraversal, cancelDuringTraversal := context.WithCancel(context.Background()) - repo.afterGet = func(id string) { - if id == "leaf" { - cancelDuringTraversal() - } - } - if _, err := resolver.GetProfile(duringTraversal, "leaf"); !errors.Is(err, context.Canceled) { - t.Fatalf("during traversal error = %v", err) - } - if got := repo.callCount("base"); got != 2 { - t.Fatalf("base calls after cancellation = %d, want 2", got) - } - - terminalLookup, cancelTerminalLookup := context.WithCancel(context.Background()) - repo.afterGet = func(id string) { - if id == "base" { - cancelTerminalLookup() - } - } - if _, err := resolver.GetProfile(terminalLookup, "leaf"); !errors.Is(err, context.Canceled) { - t.Fatalf("terminal lookup cancellation error = %v", err) - } - if got := repo.callCount("base"); got != 3 { - t.Fatalf("base calls after terminal cancellation = %d, want 3", got) - } - - repo.afterGet = nil - var wg sync.WaitGroup - errors := make(chan error, 8) - for index := 0; index < cap(errors); index++ { - wg.Add(1) - go func() { - defer wg.Done() - resolved, err := resolver.GetProfile(context.Background(), "leaf") - if err != nil { - errors <- err - return - } - resolved.ExtraParams["nested"].(map[string]any)["value"] = "mutated" - }() - } - wg.Wait() - close(errors) - for err := range errors { - t.Errorf("concurrent resolution: %v", err) - } - latest, err := resolver.GetProfile(context.Background(), "leaf") - if err != nil || latest.ExtraParams["nested"].(map[string]any)["value"] != "second" { - t.Fatalf("latest result=(%+v, %v)", latest, err) - } -} - -type resolvingTestRepository struct { - mu sync.Mutex - profiles map[string]*domain.ExecutionProfile - errors map[string]error - calls map[string]int - afterGet func(string) -} - -func (r *resolvingTestRepository) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) { - if err := ctx.Err(); err != nil { - return nil, err - } - r.mu.Lock() - if r.calls == nil { - r.calls = make(map[string]int) - } - r.calls[id]++ - err := r.errors[id] - profile := r.profiles[id] - afterGet := r.afterGet - r.mu.Unlock() - if afterGet != nil { - afterGet(id) - } - if err != nil { - return nil, err - } - if profile == nil { - if _, exists := r.profiles[id]; exists { - return nil, nil - } - return nil, ErrProfileNotFound - } - copy := *profile - return ©, nil -} - -func (r *resolvingTestRepository) set(id string, profile *domain.ExecutionProfile) { - r.mu.Lock() - defer r.mu.Unlock() - r.profiles[id] = profile -} - -func (r *resolvingTestRepository) callCount(id string) int { - r.mu.Lock() - defer r.mu.Unlock() - return r.calls[id] -} - -func profileTestFS(files map[string]string) fs.FS { - fsys := make(fstest.MapFS, len(files)) - for name, content := range files { - fsys[name] = profileMapFile(content) - } - return fsys -} diff --git a/internal/profile/testdata/invalid_yaml.yaml b/internal/profile/testdata/invalid_yaml.yaml deleted file mode 100644 index 6ba79f8..0000000 --- a/internal/profile/testdata/invalid_yaml.yaml +++ /dev/null @@ -1,3 +0,0 @@ -id: invalid_yaml -endpoint: http://localhost:8000/v1 -model: [broken diff --git a/internal/profile/testdata/missing_endpoint.yaml b/internal/profile/testdata/missing_endpoint.yaml deleted file mode 100644 index 443c8d6..0000000 --- a/internal/profile/testdata/missing_endpoint.yaml +++ /dev/null @@ -1,2 +0,0 @@ -id: missing-endpoint -model: gpt-4o-mini diff --git a/internal/profile/testdata/missing_id.yaml b/internal/profile/testdata/missing_id.yaml deleted file mode 100644 index 4c1421d..0000000 --- a/internal/profile/testdata/missing_id.yaml +++ /dev/null @@ -1,2 +0,0 @@ -endpoint: http://localhost:8000/v1 -model: gpt-4o-mini diff --git a/internal/profile/testdata/missing_model.yaml b/internal/profile/testdata/missing_model.yaml deleted file mode 100644 index 838092b..0000000 --- a/internal/profile/testdata/missing_model.yaml +++ /dev/null @@ -1,2 +0,0 @@ -id: missing-model -endpoint: http://localhost:8000/v1 diff --git a/internal/profile/testdata/raw_api_key.yaml b/internal/profile/testdata/raw_api_key.yaml deleted file mode 100644 index a1b61ea..0000000 --- a/internal/profile/testdata/raw_api_key.yaml +++ /dev/null @@ -1,4 +0,0 @@ -id: raw-api-key -endpoint: http://localhost:8000/v1 -model: gpt-4o-mini -api_key: super-secret-should-not-be-here diff --git a/internal/profile/testdata/unknown_field.yaml b/internal/profile/testdata/unknown_field.yaml deleted file mode 100644 index 9b1916a..0000000 --- a/internal/profile/testdata/unknown_field.yaml +++ /dev/null @@ -1,4 +0,0 @@ -id: unknown-field -endpoint: http://localhost:8000/v1 -model: gpt-4o-mini -foo: bar diff --git a/internal/profile/testdata/valid_local_profile.yaml b/internal/profile/testdata/valid_local_profile.yaml deleted file mode 100644 index 6430a37..0000000 --- a/internal/profile/testdata/valid_local_profile.yaml +++ /dev/null @@ -1,7 +0,0 @@ -id: local-default -endpoint: http://localhost:8000/v1 -model: gpt-4o-mini -temperature: 0.2 -max_tokens: 700 -top_p: 1.0 -timeout_seconds: 120 diff --git a/internal/profile/testdata/valid_with_api_key_env.yaml b/internal/profile/testdata/valid_with_api_key_env.yaml deleted file mode 100644 index 3bfb7d8..0000000 --- a/internal/profile/testdata/valid_with_api_key_env.yaml +++ /dev/null @@ -1,8 +0,0 @@ -id: local-secure -endpoint: http://localhost:8000/v1 -model: gpt-4o-mini -api_key_env: PROMPTKIT_API_KEY -service_tier: priority -reasoning_effort: medium -extra_params: - provider: local diff --git a/internal/promptdef/content_source.go b/internal/promptdef/content_source.go deleted file mode 100644 index 37fda81..0000000 --- a/internal/promptdef/content_source.go +++ /dev/null @@ -1,98 +0,0 @@ -package promptdef - -import ( - "fmt" - "io/fs" - "os" - "path" - "path/filepath" - "strings" - - "gitea.maximumdirect.net/eric/promptkit/internal/filecatalog" -) - -type contentSourceRoot interface { - readContentFile(sourcePath string, contentFile string) (string, string, error) -} - -type osContentSourceRoot struct { - root string - sourcePathsRelative bool -} - -func (r osContentSourceRoot) readContentFile(sourcePath string, contentFile string) (string, string, error) { - if strings.TrimSpace(contentFile) == "" { - return "", "", fmt.Errorf("path is required") - } - if filepath.IsAbs(contentFile) { - return "", "", fmt.Errorf("path %q must be relative", contentFile) - } - - root, err := filepath.Abs(r.root) - if err != nil { - return "", "", fmt.Errorf("resolve source root %q: %w", r.root, err) - } - canonicalRoot, err := filepath.EvalSymlinks(root) - if err != nil { - return "", "", fmt.Errorf("resolve source root %q: %w", r.root, err) - } - - promptPath := sourcePath - if r.sourcePathsRelative && !filepath.IsAbs(promptPath) { - promptPath = filepath.Join(root, filepath.FromSlash(promptPath)) - } else { - promptPath, err = filepath.Abs(promptPath) - if err != nil { - return "", "", fmt.Errorf("resolve prompt source %q: %w", sourcePath, err) - } - } - resolvedPath := filepath.Clean(filepath.Join(filepath.Dir(promptPath), contentFile)) - if !containsOSPath(root, resolvedPath) { - return "", "", fmt.Errorf("path %q escapes source root %q", contentFile, r.root) - } - - canonicalPath, err := filepath.EvalSymlinks(resolvedPath) - if err != nil { - return "", "", err - } - if !containsOSPath(canonicalRoot, canonicalPath) { - return "", "", fmt.Errorf("path %q escapes source root %q", contentFile, r.root) - } - - body, err := os.ReadFile(canonicalPath) - if err != nil { - return "", "", err - } - return string(body), resolvedPath, nil -} - -type fsContentSourceRoot struct { - fsys fs.FS - root string -} - -func (r fsContentSourceRoot) readContentFile(sourcePath string, contentFile string) (string, string, error) { - root := filecatalog.CleanFSRoot(r.root) - cleanSourcePath := path.Clean(sourcePath) - if cleanSourcePath == root { - root = path.Dir(root) - } - - resolvedPath, _, err := filecatalog.ResolveFSPath(root, path.Dir(cleanSourcePath), contentFile) - if err != nil { - return "", "", err - } - body, err := fs.ReadFile(r.fsys, resolvedPath) - if err != nil { - return "", "", err - } - return string(body), resolvedPath, nil -} - -func containsOSPath(root string, name string) bool { - relative, err := filepath.Rel(root, name) - if err != nil || filepath.IsAbs(relative) { - return false - } - return relative != ".." && !strings.HasPrefix(relative, ".."+string(filepath.Separator)) -} diff --git a/internal/promptdef/filesystem_repository.go b/internal/promptdef/filesystem_repository.go deleted file mode 100644 index 4080ff8..0000000 --- a/internal/promptdef/filesystem_repository.go +++ /dev/null @@ -1,328 +0,0 @@ -package promptdef - -import ( - "bytes" - "context" - "errors" - "fmt" - "io" - "io/fs" - "strings" - - "gitea.maximumdirect.net/eric/promptkit/internal/domain" - "gopkg.in/yaml.v3" -) - -var ( - ErrPromptDefinitionNotFound = errors.New("prompt definition not found") - ErrInvalidYAML = errors.New("invalid YAML format") - ErrInvalidPromptDefinition = errors.New("invalid prompt definition configuration") -) - -type sourceRepository struct { - source promptDefinitionSource -} - -type promptDefinitionFile struct { - ID string `yaml:"id"` - Version string `yaml:"version"` - DefaultProfile *string `yaml:"default_profile"` - Description string `yaml:"description"` - SessionID string `yaml:"session_id"` - Inputs []promptInputFile `yaml:"inputs"` - Messages []promptMessageFile `yaml:"messages"` - Output promptOutputContractFile `yaml:"output"` -} - -type promptInputFile struct { - Name string `yaml:"name"` - Required bool `yaml:"required"` - ContentType string `yaml:"content_type"` - Description string `yaml:"description"` -} - -type promptMessageFile struct { - Role string `yaml:"role"` - Content string `yaml:"content"` - ContentFile string `yaml:"content_file"` - CacheControl *cacheControlFile `yaml:"cache_control"` -} - -type cacheControlFile struct { - Type string `yaml:"type"` - TTL string `yaml:"ttl"` -} - -type promptOutputContractFile struct { - Format domain.OutputFormat `yaml:"format"` - ValidationMode domain.ValidationMode `yaml:"validation_mode"` - SchemaPath string `yaml:"schema_path"` - RepairAttempts int `yaml:"repair_attempts"` -} - -func NewFilesystemRepository(dir string) Repository { - return &sourceRepository{ - source: osPromptSource{ - root: dir, - contentRoot: osContentSourceRoot{root: dir}, - }, - } -} - -func NewFSRepository(fsys fs.FS, root string) Repository { - return &sourceRepository{ - source: fsPromptSource{ - fsys: fsys, - root: root, - contentRoot: fsContentSourceRoot{fsys: fsys, root: root}, - }, - } -} - -// NewFileRepository constructs a repository for one operating-system prompt file. -func NewFileRepository(fsys fs.FS, file string, sourceDir string) Repository { - return &sourceRepository{ - source: fsPromptSource{ - fsys: fsys, - root: file, - contentRoot: osContentSourceRoot{ - root: sourceDir, - sourcePathsRelative: true, - }, - }, - } -} - -func (r *sourceRepository) GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) { - if strings.TrimSpace(id) == "" { - return nil, fmt.Errorf("%w: prompt id is required", ErrInvalidPromptDefinition) - } - if r == nil || r.source == nil { - return nil, errors.New("failed to read prompt definition directory: source is nil") - } - - files, err := r.source.findYAMLFiles(ctx) - if err != nil { - return nil, fmt.Errorf("failed to read prompt definition directory: %w", err) - } - - var matches []promptDefinitionMatch - for _, fullPath := range files { - select { - case <-ctx.Done(): - return nil, ctx.Err() - default: - } - - relPath := r.source.displayPath(fullPath) - data, err := r.source.readDefinition(fullPath) - if err != nil { - return nil, fmt.Errorf("failed to read prompt definition file %s: %w", relPath, err) - } - raw, err := decodePromptDefinition(data) - if err != nil { - if promptDefinitionDataMatches(data, id, version) { - return nil, fmt.Errorf("%w: %s: %v", ErrInvalidYAML, relPath, err) - } - continue - } - if !promptDefinitionMatches(raw, id, version) { - continue - } - matches = append(matches, promptDefinitionMatch{ - raw: raw, - sourcePath: fullPath, - path: relPath, - }) - } - - if len(matches) > 1 { - paths := make([]string, 0, len(matches)) - for _, match := range matches { - paths = append(paths, match.path) - } - if version != "" { - return nil, fmt.Errorf("%w: duplicate prompt definition id %q version %q found in: %s", ErrInvalidPromptDefinition, id, version, strings.Join(paths, ", ")) - } - return nil, fmt.Errorf("%w: duplicate prompt definition id %q found in: %s", ErrInvalidPromptDefinition, id, strings.Join(paths, ", ")) - } - - if len(matches) == 1 { - match := matches[0] - def, err := normalizePromptDefinition(match.raw, r.source, match.sourcePath) - if err != nil { - return nil, fmt.Errorf("%w: %s: %v", ErrInvalidPromptDefinition, match.path, err) - } - return def, nil - } - - return nil, ErrPromptDefinitionNotFound -} - -type promptDefinitionMatch struct { - raw *promptDefinitionFile - sourcePath string - path string -} - -func decodePromptDefinition(data []byte) (*promptDefinitionFile, error) { - var raw promptDefinitionFile - decoder := yaml.NewDecoder(bytes.NewReader(data)) - decoder.KnownFields(true) - if err := decoder.Decode(&raw); err != nil { - return nil, err - } - var additional yaml.Node - if err := decoder.Decode(&additional); err != io.EOF { - if err != nil { - return nil, err - } - return nil, errors.New("prompt definition file must contain exactly one YAML document") - } - return &raw, nil -} - -func promptDefinitionDataMatches(data []byte, id string, version string) bool { - var raw struct { - ID string `yaml:"id"` - Version string `yaml:"version"` - } - if err := yaml.NewDecoder(bytes.NewReader(data)).Decode(&raw); err != nil { - return false - } - return promptSelectorMatches(raw.ID, raw.Version, id, version) -} - -func promptDefinitionMatches(raw *promptDefinitionFile, id string, version string) bool { - if raw == nil { - return false - } - return promptSelectorMatches(raw.ID, raw.Version, id, version) -} - -func promptSelectorMatches(rawID string, rawVersion string, id string, version string) bool { - if strings.TrimSpace(rawID) != id { - return false - } - return version == "" || strings.TrimSpace(rawVersion) == version -} - -func normalizePromptDefinition(raw *promptDefinitionFile, sourceRoot contentSourceRoot, sourcePath string) (*domain.PromptDefinition, error) { - return normalizePromptDefinitionWithContent(raw, func(contentFile string) (string, string, error) { - return sourceRoot.readContentFile(sourcePath, contentFile) - }) -} - -func normalizePromptDefinitionWithContent(raw *promptDefinitionFile, readContentFile func(string) (string, string, error)) (*domain.PromptDefinition, error) { - if raw == nil { - return nil, errors.New("prompt definition is nil") - } - - id := strings.TrimSpace(raw.ID) - if id == "" { - return nil, errors.New("id is required") - } - - version := strings.TrimSpace(raw.Version) - if version == "" { - return nil, errors.New("version is required") - } - - if len(raw.Messages) == 0 { - return nil, errors.New("at least one message is required") - } - - inputs := make([]domain.PromptInput, 0, len(raw.Inputs)) - seenInputNames := make(map[string]struct{}, len(raw.Inputs)) - for i, in := range raw.Inputs { - name := strings.TrimSpace(in.Name) - if name == "" { - return nil, fmt.Errorf("input %d has empty name", i) - } - if _, exists := seenInputNames[name]; exists { - return nil, fmt.Errorf("duplicate input name %q", name) - } - seenInputNames[name] = struct{}{} - - inputs = append(inputs, domain.PromptInput{ - Name: name, - Required: in.Required, - ContentType: strings.TrimSpace(in.ContentType), - Description: strings.TrimSpace(in.Description), - }) - } - - templates := make([]domain.PromptMessageTemplate, 0, len(raw.Messages)) - for i, msg := range raw.Messages { - role, err := domain.NormalizeMessageRole(msg.Role) - if err != nil { - return nil, fmt.Errorf("message %d role: %w", i, err) - } - - hasContent := strings.TrimSpace(msg.Content) != "" - hasContentFile := strings.TrimSpace(msg.ContentFile) != "" - if hasContent == hasContentFile { - return nil, fmt.Errorf("message %d must set exactly one of content or content_file", i) - } - - var rawCacheControl *domain.CacheControl - if msg.CacheControl != nil { - rawCacheControl = &domain.CacheControl{ - Type: domain.CacheControlType(msg.CacheControl.Type), - TTL: msg.CacheControl.TTL, - } - } - cacheControl, err := domain.NormalizeCacheControl(rawCacheControl) - if err != nil { - return nil, fmt.Errorf("message %d cache_control: %w", i, err) - } - - templateContent := msg.Content - resolvedContentFile := "" - if hasContentFile { - body, resolvedPath, err := readContentFile(msg.ContentFile) - if err != nil { - return nil, fmt.Errorf("prompt %q message %d: failed to read content_file %q: %w", id, i, msg.ContentFile, err) - } - templateContent = body - resolvedContentFile = resolvedPath - } - - templates = append(templates, domain.PromptMessageTemplate{ - Role: role, - Content: templateContent, - ContentFile: resolvedContentFile, - CacheControl: cacheControl, - }) - } - - outputContract := domain.OutputContract{ - Format: raw.Output.Format, - ValidationMode: raw.Output.ValidationMode, - SchemaPath: strings.TrimSpace(raw.Output.SchemaPath), - RepairAttempts: raw.Output.RepairAttempts, - } - if err := domain.ValidateOutputContract(outputContract); err != nil { - return nil, fmt.Errorf("output: %w", err) - } - - defaultProfile := "" - if raw.DefaultProfile != nil { - defaultProfile = strings.TrimSpace(*raw.DefaultProfile) - if defaultProfile == "" { - return nil, errors.New("default_profile must be a non-empty string when set") - } - } - - return &domain.PromptDefinition{ - ID: id, - Version: version, - DefaultProfile: defaultProfile, - Description: strings.TrimSpace(raw.Description), - SessionID: strings.TrimSpace(raw.SessionID), - Inputs: inputs, - Templates: templates, - OutputFormat: raw.Output.Format, - Validation: outputContract, - }, nil -} diff --git a/internal/promptdef/repository.go b/internal/promptdef/repository.go index b1241d4..67a4aa1 100644 --- a/internal/promptdef/repository.go +++ b/internal/promptdef/repository.go @@ -2,11 +2,15 @@ package promptdef import ( "context" + "errors" "gitea.maximumdirect.net/eric/promptkit/internal/domain" ) -// Repository loads prompt definitions. +// ErrPromptDefinitionNotFound identifies an absent captured prompt definition. +var ErrPromptDefinitionNotFound = errors.New("prompt definition not found") + +// Repository selects prompt definitions from a captured index. type Repository interface { GetPromptDefinition(ctx context.Context, id string, version string) (*domain.PromptDefinition, error) } diff --git a/internal/promptdef/repository_test.go b/internal/promptdef/repository_test.go deleted file mode 100644 index c332d95..0000000 --- a/internal/promptdef/repository_test.go +++ /dev/null @@ -1,1176 +0,0 @@ -package promptdef - -import ( - "context" - "errors" - "fmt" - "io/fs" - "os" - "path/filepath" - "strings" - "sync" - "testing" - "testing/fstest" - - "gitea.maximumdirect.net/eric/promptkit/internal/domain" -) - -func TestPromptRepositoryDefinitionFixtures(t *testing.T) { - sources := []struct { - name string - newRepository func(string) Repository - contentPathsAreFull bool - }{ - { - name: "operating system", - newRepository: NewFilesystemRepository, - contentPathsAreFull: true, - }, - { - name: "filesystem", - newRepository: func(root string) Repository { - return NewFSRepository(os.DirFS(root), ".") - }, - }, - } - - for _, source := range sources { - t.Run(source.name, func(t *testing.T) { - testPromptRepositoryDefinitionFixtures(t, source.newRepository, source.contentPathsAreFull) - }) - } -} - -func testPromptRepositoryDefinitionFixtures(t *testing.T, newRepository func(string) Repository, contentPathsAreFull bool) { - tmpDir := t.TempDir() - if err := copyTree("testdata", tmpDir); err != nil { - t.Fatalf("failed to copy testdata: %v", err) - } - - repo := newRepository(tmpDir) - ctx := context.Background() - - t.Run("valid inline prompt", func(t *testing.T) { - p, err := repo.GetPromptDefinition(ctx, "valid-inline", "") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if p.ID != "valid-inline" { - t.Fatalf("unexpected id: %q", p.ID) - } - if p.Version != "1.0.0" { - t.Fatalf("unexpected version: %q", p.Version) - } - if p.OutputFormat != domain.FormatMarkdown { - t.Fatalf("unexpected output format: %q", p.OutputFormat) - } - if p.Validation.ValidationMode != domain.ValidationBasic { - t.Fatalf("unexpected validation mode: %q", p.Validation.ValidationMode) - } - if len(p.Templates) != 2 { - t.Fatalf("expected 2 messages, got %d", len(p.Templates)) - } - if len(p.Inputs) != 1 { - t.Fatalf("expected 1 input, got %d", len(p.Inputs)) - } - if p.Inputs[0].ContentType != "text/markdown" { - t.Fatalf("expected input content_type to be preserved, got %q", p.Inputs[0].ContentType) - } - }) - - t.Run("valid file-backed prompt", func(t *testing.T) { - p, err := repo.GetPromptDefinition(ctx, "valid-file-backed", "") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if len(p.Templates) != 2 { - t.Fatalf("expected 2 messages, got %d", len(p.Templates)) - } - if !strings.Contains(p.Templates[1].Content, "{{input \"transcript\"}}") { - t.Fatalf("expected content_file template body to be loaded, got %q", p.Templates[1].Content) - } - if p.Templates[1].ContentFile == "" { - t.Fatal("expected ContentFile source metadata to be preserved") - } - if filepath.IsAbs(p.Templates[1].ContentFile) != contentPathsAreFull { - t.Fatalf("unexpected content_file path representation: %q", p.Templates[1].ContentFile) - } - }) - - t.Run("valid cache control with ttl", func(t *testing.T) { - p, err := repo.GetPromptDefinition(ctx, "valid-cache-control-ttl", "") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if len(p.Templates) != 2 { - t.Fatalf("expected 2 messages, got %d", len(p.Templates)) - } - assertCacheControl(t, p.Templates[0].CacheControl, domain.CacheControlEphemeral, "1h") - if p.Templates[1].CacheControl != nil { - t.Fatalf("expected second message cache control to be nil, got %#v", p.Templates[1].CacheControl) - } - }) - - t.Run("valid cache control without ttl", func(t *testing.T) { - p, err := repo.GetPromptDefinition(ctx, "valid-cache-control-without-ttl", "") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if len(p.Templates) != 2 { - t.Fatalf("expected 2 messages, got %d", len(p.Templates)) - } - assertCacheControl(t, p.Templates[0].CacheControl, domain.CacheControlEphemeral, "") - if p.Templates[1].CacheControl != nil { - t.Fatalf("expected second message cache control to be nil, got %#v", p.Templates[1].CacheControl) - } - }) - - t.Run("valid session id template", func(t *testing.T) { - p, err := repo.GetPromptDefinition(ctx, "valid-session-id", "") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if p.SessionID != "{{ .session_id }}" { - t.Fatalf("expected trimmed session_id template, got %q", p.SessionID) - } - }) - - t.Run("valid nested file-backed prompt resolves content file relative to nested YAML", func(t *testing.T) { - nestedDir := filepath.Join(tmpDir, "dnd", "recap") - if err := os.MkdirAll(nestedDir, 0o755); err != nil { - t.Fatal(err) - } - writePromptTestFile(t, filepath.Join(nestedDir, "nested_recap.yaml"), ` -id: nested-recap -version: "1.0.0" -messages: - - role: user - content_file: ./nested_recap.user.tmpl -output: - format: markdown - validation_mode: basic - repair_attempts: 0 -`) - writePromptTestFile(t, filepath.Join(nestedDir, "nested_recap.user.tmpl"), `Nested recap: {{input "transcript"}}`) - - p, err := repo.GetPromptDefinition(ctx, "nested-recap", "") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if len(p.Templates) != 1 { - t.Fatalf("expected one template, got %d", len(p.Templates)) - } - if !strings.Contains(p.Templates[0].Content, "Nested recap") { - t.Fatalf("expected nested content file body, got %q", p.Templates[0].Content) - } - if !strings.Contains(p.Templates[0].ContentFile, filepath.Join("dnd", "recap", "nested_recap.user.tmpl")) { - t.Fatalf("expected nested content file path, got %q", p.Templates[0].ContentFile) - } - }) - - t.Run("prompt with default_profile", func(t *testing.T) { - p, err := repo.GetPromptDefinition(ctx, "with-default-profile", "") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if p.DefaultProfile != "local-default" { - t.Fatalf("unexpected default profile: %q", p.DefaultProfile) - } - if len(p.Inputs) != 1 { - t.Fatalf("expected one input, got %d", len(p.Inputs)) - } - if p.Inputs[0].ContentType != "" { - t.Fatalf("expected missing content_type to remain empty, got %q", p.Inputs[0].ContentType) - } - }) - - t.Run("duplicate prompt IDs fail as ambiguous", func(t *testing.T) { - writePromptTestFile(t, filepath.Join(tmpDir, "duplicate_a.yaml"), ` -id: duplicate-prompt -version: "1.0.0" -messages: - - role: user - content: First duplicate. -output: - format: markdown - validation_mode: basic - repair_attempts: 0 -`) - nestedDir := filepath.Join(tmpDir, "nested") - if err := os.MkdirAll(nestedDir, 0o755); err != nil { - t.Fatal(err) - } - writePromptTestFile(t, filepath.Join(nestedDir, "duplicate_b.yaml"), ` -id: duplicate-prompt -version: "2.0.0" -messages: - - role: user - content: Second duplicate. -output: - format: markdown - validation_mode: basic - repair_attempts: 0 -`) - - _, err := repo.GetPromptDefinition(ctx, "duplicate-prompt", "") - if !errors.Is(err, ErrInvalidPromptDefinition) { - t.Fatalf("expected duplicate prompt to return ErrInvalidPromptDefinition, got %v", err) - } - for _, want := range []string{"duplicate prompt definition id", "duplicate_a.yaml", filepath.Join("nested", "duplicate_b.yaml")} { - if !strings.Contains(err.Error(), want) { - t.Fatalf("expected error to contain %q, got %v", want, err) - } - } - }) - - t.Run("duplicate prompt ID and requested version fails as ambiguous", func(t *testing.T) { - writePromptTestFile(t, filepath.Join(tmpDir, "version_duplicate_a.yaml"), ` -id: duplicate-version-prompt -version: "1.0.0" -messages: - - role: user - content: First duplicate version. -output: - format: markdown - validation_mode: basic - repair_attempts: 0 -`) - nestedDir := filepath.Join(tmpDir, "versioned") - if err := os.MkdirAll(nestedDir, 0o755); err != nil { - t.Fatal(err) - } - writePromptTestFile(t, filepath.Join(nestedDir, "version_duplicate_b.yaml"), ` -id: duplicate-version-prompt -version: "1.0.0" -messages: - - role: user - content: Second duplicate version. -output: - format: markdown - validation_mode: basic - repair_attempts: 0 -`) - - _, err := repo.GetPromptDefinition(ctx, "duplicate-version-prompt", "1.0.0") - if !errors.Is(err, ErrInvalidPromptDefinition) { - t.Fatalf("expected duplicate prompt version to return ErrInvalidPromptDefinition, got %v", err) - } - for _, want := range []string{"duplicate prompt definition id", "version \"1.0.0\"", "version_duplicate_a.yaml", filepath.Join("versioned", "version_duplicate_b.yaml")} { - if !strings.Contains(err.Error(), want) { - t.Fatalf("expected error to contain %q, got %v", want, err) - } - } - }) - - t.Run("non-matching malformed nested prompt is ignored for not found lookup", func(t *testing.T) { - nestedDir := filepath.Join(tmpDir, "broken") - if err := os.MkdirAll(nestedDir, 0o755); err != nil { - t.Fatal(err) - } - writePromptTestFile(t, filepath.Join(nestedDir, "unrelated.yaml"), "id: [") - - _, err := repo.GetPromptDefinition(ctx, "does-not-exist-even-with-broken-nested-file", "") - if !errors.Is(err, ErrPromptDefinitionNotFound) { - t.Fatalf("expected ErrPromptDefinitionNotFound, got %v", err) - } - }) - - t.Run("strict decode failure in nested prompt matches by YAML ID", func(t *testing.T) { - nestedDir := filepath.Join(tmpDir, "strict") - if err := os.MkdirAll(nestedDir, 0o755); err != nil { - t.Fatal(err) - } - writePromptTestFile(t, filepath.Join(nestedDir, "not_named_like_id.yaml"), ` -id: nested-strict-error -version: "1.0.0" -unknown_field: true -messages: - - role: user - content: Invalid because of unknown field. -output: - format: markdown - validation_mode: basic - repair_attempts: 0 -`) - - _, err := repo.GetPromptDefinition(ctx, "nested-strict-error", "") - if !errors.Is(err, ErrInvalidYAML) { - t.Fatalf("expected ErrInvalidYAML, got %v", err) - } - if !strings.Contains(err.Error(), filepath.Join("strict", "not_named_like_id.yaml")) { - t.Fatalf("expected nested path in error, got %v", err) - } - }) - - t.Run("version lookup", func(t *testing.T) { - _, err := repo.GetPromptDefinition(ctx, "valid-inline", "9.9.9") - if !errors.Is(err, ErrPromptDefinitionNotFound) { - t.Fatalf("expected ErrPromptDefinitionNotFound, got %v", err) - } - }) - - cases := []struct { - name string - id string - targetErr error - errSubstrs []string - }{ - {name: "unidentifiable invalid YAML is unrelated", id: "invalid-yaml", targetErr: ErrPromptDefinitionNotFound}, - {name: "missing id is not selected by filename", id: "missing_id", targetErr: ErrPromptDefinitionNotFound}, - {name: "no messages", id: "no-messages", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"at least one message is required"}}, - {name: "both content and content_file", id: "both-content-and-content-file", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"exactly one"}}, - {name: "neither content nor content_file", id: "neither-content-nor-content-file", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"exactly one"}}, - {name: "missing content_file", id: "missing-content-file", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"failed to read content_file"}}, - {name: "duplicate input names", id: "duplicate-input-names", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"duplicate input name"}}, - {name: "invalid validation mode", id: "invalid-validation-mode", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"invalid validation mode"}}, - {name: "json_schema without schema_path", id: "json-schema-without-schema-path", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"schema_path"}}, - {name: "unknown input field", id: "unknown-input-field", targetErr: ErrInvalidYAML, errSubstrs: []string{"field unknown_input_setting not found"}}, - {name: "empty cache control type", id: "empty-cache-control-type", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "type is required"}}, - {name: "unsupported cache control type", id: "unsupported-cache-control-type", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "unsupported type"}}, - {name: "unsupported cache control ttl", id: "unsupported-cache-control-ttl", targetErr: ErrInvalidPromptDefinition, errSubstrs: []string{"cache_control", "unsupported ttl"}}, - {name: "unknown cache control field", id: "unknown-cache-control-field", targetErr: ErrInvalidYAML, errSubstrs: []string{"field unexpected not found"}}, - } - - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - _, err := repo.GetPromptDefinition(ctx, tc.id, "") - if !errors.Is(err, tc.targetErr) { - t.Fatalf("expected %v, got %v", tc.targetErr, err) - } - for _, sub := range tc.errSubstrs { - if !strings.Contains(err.Error(), sub) { - t.Fatalf("expected error to contain %q, got %v", sub, err) - } - } - }) - } - - t.Run("prompt definition not found", func(t *testing.T) { - _, err := repo.GetPromptDefinition(ctx, "does-not-exist", "") - if !errors.Is(err, ErrPromptDefinitionNotFound) { - t.Fatalf("expected ErrPromptDefinitionNotFound, got %v", err) - } - }) -} - -func TestFSRepositoryGetPromptDefinition(t *testing.T) { - repo := NewFSRepository(fstest.MapFS{ - "prompts/nested/prompt.yaml": &fstest.MapFile{Data: []byte(` -id: fs-prompt -version: "1.0.0" -inputs: - - name: transcript - required: true -messages: - - role: user - content_file: ./messages/user.tmpl -output: - format: markdown - validation_mode: basic - repair_attempts: 0 -`)}, - "prompts/nested/messages/user.tmpl": &fstest.MapFile{Data: []byte(`Summarize {{input "transcript"}}.`)}, - }, "prompts") - - got, err := repo.GetPromptDefinition(context.Background(), "fs-prompt", "") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if got.ID != "fs-prompt" { - t.Fatalf("unexpected prompt id: %q", got.ID) - } - if len(got.Templates) != 1 || !strings.Contains(got.Templates[0].Content, `{{input "transcript"}}`) { - t.Fatalf("expected content_file body to be loaded, got %+v", got.Templates) - } - if got.Templates[0].ContentFile != "prompts/nested/messages/user.tmpl" { - t.Fatalf("unexpected content file path: %q", got.Templates[0].ContentFile) - } -} - -func TestFSRepositoryContentFileContainment(t *testing.T) { - t.Run("nested prompt can reference file inside root", func(t *testing.T) { - repo := NewFSRepository(fstest.MapFS{ - "prompts/nested/prompt.yaml": &fstest.MapFile{Data: []byte(` -id: fs-contained-prompt -version: "1.0.0" -messages: - - role: user - content_file: ../shared/user.tmpl -output: - format: markdown - validation_mode: basic - repair_attempts: 0 -`)}, - "prompts/shared/user.tmpl": &fstest.MapFile{Data: []byte(`Inside root.`)}, - }, "prompts") - - got, err := repo.GetPromptDefinition(context.Background(), "fs-contained-prompt", "") - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if len(got.Templates) != 1 || got.Templates[0].Content != "Inside root." { - t.Fatalf("expected contained content file, got %+v", got.Templates) - } - }) - - tests := []struct { - name string - contentFile string - wantErr string - }{ - {name: "parent escape rejected", contentFile: "../outside.tmpl", wantErr: "escapes source root"}, - {name: "absolute path rejected", contentFile: "/outside.tmpl", wantErr: "must be relative"}, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - fsys := &recordingFS{FS: fstest.MapFS{ - "prompts/prompt.yaml": &fstest.MapFile{Data: []byte(` -id: fs-escaped-prompt -version: "1.0.0" -messages: - - role: user - content_file: ` + tc.contentFile + ` -output: - format: markdown - validation_mode: basic - repair_attempts: 0 -`)}, - "outside.tmpl": &fstest.MapFile{Data: []byte(`Outside root.`)}, - }} - repo := NewFSRepository(fsys, "prompts") - - _, err := repo.GetPromptDefinition(context.Background(), "fs-escaped-prompt", "") - if !errors.Is(err, ErrInvalidPromptDefinition) { - t.Fatalf("expected ErrInvalidPromptDefinition, got %v", err) - } - if !strings.Contains(err.Error(), tc.wantErr) { - t.Fatalf("expected error to contain %q, got %v", tc.wantErr, err) - } - if fsys.wasOpened("outside.tmpl") { - t.Fatal("rejected content path opened the outside file") - } - }) - } -} - -type recordingFS struct { - fs.FS - mu sync.Mutex - opened []string -} - -func TestPromptRepositoryReturnsDefinitionReadFailures(t *testing.T) { - readErr := errors.New("definition read failed") - fsys := &definitionReadFailureFS{ - FS: fstest.MapFS{ - "prompts/target.yaml": &fstest.MapFile{Data: []byte("unread")}, - }, - target: "prompts/target.yaml", - err: readErr, - } - repo := NewFSRepository(fsys, "prompts") - - definition, err := repo.GetPromptDefinition(context.Background(), "target", "1") - if definition != nil || !errors.Is(err, readErr) { - t.Fatalf("GetPromptDefinition() = (%#v, %v), want nil and definition read error", definition, err) - } - if errors.Is(err, ErrPromptDefinitionNotFound) { - t.Fatalf("definition read error was classified as absence: %v", err) - } - if !strings.Contains(err.Error(), "target.yaml") { - t.Fatalf("definition read error lacks source context: %v", err) - } -} - -type definitionReadFailureFS struct { - fs.FS - target string - err error -} - -func (f *definitionReadFailureFS) Open(name string) (fs.File, error) { - if name == f.target { - return nil, f.err - } - return f.FS.Open(name) -} - -func (f *recordingFS) Open(name string) (fs.File, error) { - f.mu.Lock() - f.opened = append(f.opened, name) - f.mu.Unlock() - return f.FS.Open(name) -} - -func (f *recordingFS) wasOpened(name string) bool { - return f.openCount(name) > 0 -} - -func (f *recordingFS) openCount(name string) int { - f.mu.Lock() - defer f.mu.Unlock() - count := 0 - for _, opened := range f.opened { - if opened == name { - count++ - } - } - return count -} - -func TestPromptRepositorySelectionUsesYAMLMetadata(t *testing.T) { - const validDefinition = ` -id: selected-prompt -version: "1" -messages: - - role: user - content: selected -output: - format: text - validation_mode: none -` - tests := []struct { - name string - files map[string]string - wantErr error - diagnostics []string - wantContent string - }{ - { - name: "same-stem strict error with different YAML ID is unrelated", - files: map[string]string{ - "selected-prompt.yaml": ` -id: another-prompt -version: "1" -unknown: true -`, - "valid.yaml": validDefinition, - }, - }, - { - name: "unidentifiable same-stem YAML is unrelated", - files: map[string]string{ - "selected-prompt.yaml": "id: [", - "valid.yaml": validDefinition, - }, - }, - { - name: "same ID invalid different version is unrelated", - files: map[string]string{ - "invalid-version.yaml": ` -id: selected-prompt -version: "2" -output: - format: text - validation_mode: none -`, - "valid.yaml": validDefinition, - }, - }, - { - name: "selected content is resolved relative to its definition", - files: map[string]string{ - "nested/selected.yaml": ` -id: selected-prompt -version: "1" -messages: - - role: user - content_file: ./content/selected.tmpl -output: - format: text - validation_mode: none -`, - "nested/content/selected.tmpl": "selected from file", - }, - wantContent: "selected from file", - }, - { - name: "duplicate selected definitions are ambiguous", - files: map[string]string{ - "selected-a.yaml": validDefinition, - "nested/selected-b.yaml": ` -id: selected-prompt -version: "1" -messages: - - role: user - content: duplicate -output: - format: text - validation_mode: none -`, - }, - wantErr: ErrInvalidPromptDefinition, - diagnostics: []string{"duplicate prompt definition id", "selected-a.yaml", "nested/selected-b.yaml"}, - }, - { - name: "selected strict error is authoritative", - files: map[string]string{ - "selected-strict.yaml": ` -id: selected-prompt -version: "1" -unknown: true -`, - }, - wantErr: ErrInvalidYAML, - diagnostics: []string{"selected-strict.yaml"}, - }, - { - name: "selected semantic error is authoritative", - files: map[string]string{ - "selected-invalid.yaml": ` -id: selected-prompt -version: "1" -output: - format: text - validation_mode: none -`, - }, - wantErr: ErrInvalidPromptDefinition, - diagnostics: []string{"selected-invalid.yaml", "at least one message"}, - }, - { - name: "selected content error includes definition context", - files: map[string]string{ - "selected-missing-content.yaml": ` -id: selected-prompt -version: "1" -messages: - - role: user - content_file: missing.tmpl -output: - format: text - validation_mode: none -`, - }, - wantErr: ErrInvalidPromptDefinition, - diagnostics: []string{"selected-missing-content.yaml", "failed to read content_file", "missing.tmpl"}, - }, - } - - for _, source := range promptRepositorySources() { - for _, tc := range tests { - t.Run(source.name+"/"+tc.name, func(t *testing.T) { - repo := source.newRepository(t, tc.files) - got, err := repo.GetPromptDefinition(context.Background(), "selected-prompt", "1") - if tc.wantErr != nil { - if !errors.Is(err, tc.wantErr) { - t.Fatalf("expected %v, got %v", tc.wantErr, err) - } - for _, diagnostic := range tc.diagnostics { - if !strings.Contains(err.Error(), diagnostic) { - t.Fatalf("expected error to contain %q, got %v", diagnostic, err) - } - } - return - } - if err != nil { - t.Fatalf("load selected prompt: %v", err) - } - wantContent := tc.wantContent - if wantContent == "" { - wantContent = "selected" - } - if got.ID != "selected-prompt" || got.Version != "1" || got.Templates[0].Content != wantContent { - t.Fatalf("unexpected selected definition: %+v", got) - } - }) - } - } -} - -func TestPromptRepositoryHonorsCancellation(t *testing.T) { - for _, source := range promptRepositorySources() { - t.Run(source.name, func(t *testing.T) { - repo := source.newRepository(t, map[string]string{ - "definition.yaml": ` -id: cancelled-prompt -version: "1" -messages: - - role: user - content: selected -output: - format: text - validation_mode: none -`, - }) - ctx, cancel := context.WithCancel(context.Background()) - cancel() - - _, err := repo.GetPromptDefinition(ctx, "cancelled-prompt", "1") - if !errors.Is(err, context.Canceled) { - t.Fatalf("expected context cancellation, got %v", err) - } - }) - } -} - -func TestPromptRepositoryRequiresOneYAMLDocument(t *testing.T) { - const definition = ` -id: one-document -version: "1" -messages: - - role: user - content: selected -output: - format: text - validation_mode: none -` - tests := []struct { - name string - suffix string - wantErr bool - }{ - {name: "comments and trailing whitespace", suffix: "\n# trailing comment\n\n"}, - {name: "second populated document", suffix: "\n---\nid: another\n", wantErr: true}, - {name: "second empty document", suffix: "\n---\n", wantErr: true}, - {name: "malformed trailing YAML", suffix: "\n---\n[", wantErr: true}, - } - - for _, source := range promptRepositorySources() { - for _, tc := range tests { - t.Run(source.name+"/"+tc.name, func(t *testing.T) { - repo := source.newRepository(t, map[string]string{"definition.yaml": definition + tc.suffix}) - _, err := repo.GetPromptDefinition(context.Background(), "one-document", "1") - if tc.wantErr { - if !errors.Is(err, ErrInvalidYAML) { - t.Fatalf("expected ErrInvalidYAML, got %v", err) - } - if !strings.Contains(err.Error(), "definition.yaml") { - t.Fatalf("expected source path in error, got %v", err) - } - return - } - if err != nil { - t.Fatalf("load one document: %v", err) - } - }) - } - } -} - -func TestPromptRepositoryReadsOnlySelectedContent(t *testing.T) { - fsys := &recordingFS{FS: fstest.MapFS{ - "target.yaml": &fstest.MapFile{Data: []byte(` -id: target -version: "1" -messages: - - role: user - content_file: target.tmpl -output: - format: text - validation_mode: none -`)}, - "target.tmpl": &fstest.MapFile{Data: []byte("selected")}, - "unrelated.yaml": &fstest.MapFile{Data: []byte(` -id: unrelated -version: "1" -messages: - - role: user - content_file: unrelated.tmpl -output: - format: text - validation_mode: none -`)}, - "unrelated.tmpl": &fstest.MapFile{Data: []byte("unrelated")}, - "other-version.yaml": &fstest.MapFile{Data: []byte(` -id: target -version: "2" -messages: - - role: user - content_file: other-version.tmpl -output: - format: text - validation_mode: none -`)}, - "other-version.tmpl": &fstest.MapFile{Data: []byte("other version")}, - }} - repo := NewFSRepository(fsys, ".") - - for lookup := 1; lookup <= 2; lookup++ { - got, err := repo.GetPromptDefinition(context.Background(), "target", "1") - if err != nil { - t.Fatalf("lookup %d: %v", lookup, err) - } - if got.Templates[0].Content != "selected" { - t.Fatalf("lookup %d content = %q", lookup, got.Templates[0].Content) - } - if count := fsys.openCount("target.tmpl"); count != lookup { - t.Fatalf("selected content opens after lookup %d = %d, want %d", lookup, count, lookup) - } - for _, name := range []string{"unrelated.tmpl", "other-version.tmpl"} { - if count := fsys.openCount(name); count != 0 { - t.Fatalf("unselected content %q opened %d times", name, count) - } - } - for _, name := range []string{"target.yaml", "unrelated.yaml", "other-version.yaml"} { - if count := fsys.openCount(name); count != lookup { - t.Fatalf("metadata %q opens after lookup %d = %d, want %d", name, lookup, count, lookup) - } - } - } -} - -func TestPromptDefinitionNormalizationRules(t *testing.T) { - tests := []struct { - name string - definition string - wantErr bool - wantDiagnostic string - wantSchemaPath string - }{ - { - name: "missing version", - definition: ` -id: normalization-rule -messages: - - role: user - content: test -output: - format: text - validation_mode: none -`, - wantErr: true, - wantDiagnostic: "version", - }, - { - name: "blank input name", - definition: ` -id: normalization-rule -version: "1" -inputs: - - name: " " -messages: - - role: user - content: test -output: - format: text - validation_mode: none -`, - wantErr: true, - wantDiagnostic: "input 0", - }, - { - name: "blank message role", - definition: ` -id: normalization-rule -version: "1" -messages: - - role: " " - content: test -output: - format: text - validation_mode: none -`, - wantErr: true, - wantDiagnostic: "role", - }, - { - name: "invalid output format", - definition: ` -id: normalization-rule -version: "1" -messages: - - role: user - content: test -output: - format: binary - validation_mode: none -`, - wantErr: true, - wantDiagnostic: "format", - }, - { - name: "negative repair attempts", - definition: ` -id: normalization-rule -version: "1" -messages: - - role: user - content: test -output: - format: text - validation_mode: none - repair_attempts: -1 -`, - wantErr: true, - wantDiagnostic: "repair_attempts", - }, - { - name: "repair attempts above maximum", - definition: ` -id: normalization-rule -version: "1" -messages: - - role: user - content: test -output: - format: text - validation_mode: basic - repair_attempts: 4 -`, - wantErr: true, - wantDiagnostic: "repair_attempts", - }, - { - name: "none validation with repair attempts", - definition: ` -id: normalization-rule -version: "1" -messages: - - role: user - content: test -output: - format: text - validation_mode: none - repair_attempts: 1 -`, - wantErr: true, - wantDiagnostic: "repair_attempts", - }, - { - name: "explicit blank default profile", - definition: ` -id: normalization-rule -version: "1" -default_profile: " " -messages: - - role: user - content: test -output: - format: text - validation_mode: none -`, - wantErr: true, - wantDiagnostic: "default_profile", - }, - { - name: "schema path normalization", - definition: ` -id: normalization-rule -version: "1" -messages: - - role: user - content: test -output: - format: json - validation_mode: json_schema - schema_path: ' schema.json ' -`, - wantSchemaPath: "schema.json", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - repo := NewFSRepository(fstest.MapFS{ - "definition.yaml": &fstest.MapFile{Data: []byte(tt.definition)}, - }, ".") - - got, err := repo.GetPromptDefinition(context.Background(), "normalization-rule", "") - if tt.wantErr { - if !errors.Is(err, ErrInvalidPromptDefinition) { - t.Fatalf("expected ErrInvalidPromptDefinition, got %v", err) - } - if !strings.Contains(err.Error(), tt.wantDiagnostic) { - t.Fatalf("expected error containing %q, got %v", tt.wantDiagnostic, err) - } - return - } - if err != nil { - t.Fatalf("load prompt definition: %v", err) - } - if got.Validation.SchemaPath != tt.wantSchemaPath { - t.Fatalf("schema path = %q, want %q", got.Validation.SchemaPath, tt.wantSchemaPath) - } - }) - } -} - -type promptRepositorySource struct { - name string - newRepository func(t *testing.T, files map[string]string) Repository -} - -func promptRepositorySources() []promptRepositorySource { - return []promptRepositorySource{ - { - name: "operating system", - newRepository: func(t *testing.T, files map[string]string) Repository { - t.Helper() - root := t.TempDir() - for name, content := range files { - filePath := filepath.Join(root, filepath.FromSlash(name)) - if err := os.MkdirAll(filepath.Dir(filePath), 0o755); err != nil { - t.Fatalf("create prompt directory: %v", err) - } - writePromptTestFile(t, filePath, content) - } - return NewFilesystemRepository(root) - }, - }, - { - name: "filesystem", - newRepository: func(t *testing.T, files map[string]string) Repository { - t.Helper() - fsys := make(fstest.MapFS, len(files)) - for name, content := range files { - fsys[name] = &fstest.MapFile{Data: []byte(strings.TrimLeft(content, "\n"))} - } - return NewFSRepository(fsys, ".") - }, - }, - } -} - -func BenchmarkPromptRepositoryLookup(b *testing.B) { - for _, size := range []int{10, 1000} { - b.Run(fmt.Sprintf("catalog-%d", size), func(b *testing.B) { - files := fstest.MapFS{ - "target.yaml": &fstest.MapFile{Data: []byte(` -id: target -version: "1" -messages: - - role: user - content_file: target.tmpl -output: - format: text - validation_mode: none -`)}, - "target.tmpl": &fstest.MapFile{Data: []byte("selected")}, - } - metadataNames := []string{"target.yaml"} - contentNames := make([]string, 0, size-1) - for i := 1; i < size; i++ { - definitionName := fmt.Sprintf("prompt-%04d.yaml", i) - contentName := fmt.Sprintf("prompt-%04d.tmpl", i) - files[definitionName] = &fstest.MapFile{Data: []byte(fmt.Sprintf(` -id: prompt-%04d -version: "1" -messages: - - role: user - content_file: %s -output: - format: text - validation_mode: none -`, i, contentName))} - files[contentName] = &fstest.MapFile{Data: []byte("unrelated")} - metadataNames = append(metadataNames, definitionName) - contentNames = append(contentNames, contentName) - } - - fsys := &recordingFS{FS: files} - repo := NewFSRepository(fsys, ".") - b.ReportAllocs() - b.ResetTimer() - for i := 0; i < b.N; i++ { - if _, err := repo.GetPromptDefinition(context.Background(), "target", "1"); err != nil { - b.Fatal(err) - } - } - b.StopTimer() - - if got := fsys.openCount("target.tmpl"); got != b.N { - b.Fatalf("selected content opens = %d, want %d", got, b.N) - } - for _, name := range contentNames { - if got := fsys.openCount(name); got != 0 { - b.Fatalf("unrelated content %q opened %d times", name, got) - } - } - for _, name := range metadataNames { - if got := fsys.openCount(name); got != b.N { - b.Fatalf("metadata %q opens = %d, want %d", name, got, b.N) - } - } - }) - } -} - -func TestPromptDefinitionMessageRoleNormalization(t *testing.T) { - const invalidRole = "consumer-private-role" - repo := NewFSRepository(fstest.MapFS{ - "canonical.yaml": {Data: []byte(` -id: canonical -version: "1" -messages: - - role: " \u2003SyStEm\u2003 " - content: test -output: - format: text - validation_mode: none -`)}, - "invalid.yaml": {Data: []byte(` -id: invalid-role -version: "1" -messages: - - role: consumer-private-role - content: test -output: - format: text - validation_mode: none -`)}, - }, ".") - - definition, err := repo.GetPromptDefinition(context.Background(), "canonical", "") - if err != nil { - t.Fatalf("GetPromptDefinition() error = %v", err) - } - if got := definition.Templates[0].Role; got != domain.RoleSystem { - t.Fatalf("normalized role = %q, want %q", got, domain.RoleSystem) - } - - _, err = repo.GetPromptDefinition(context.Background(), "invalid-role", "") - if !errors.Is(err, ErrInvalidPromptDefinition) { - t.Fatalf("expected ErrInvalidPromptDefinition, got %v", err) - } - if strings.Contains(err.Error(), invalidRole) { - t.Fatalf("invalid prompt error exposed the supplied role: %v", err) - } -} - -func assertCacheControl(t *testing.T, got *domain.CacheControl, wantType domain.CacheControlType, wantTTL string) { - t.Helper() - if got == nil { - t.Fatal("expected cache control, got nil") - } - if got.Type != wantType { - t.Fatalf("unexpected cache control type: got %q want %q", got.Type, wantType) - } - if got.TTL != wantTTL { - t.Fatalf("unexpected cache control ttl: got %q want %q", got.TTL, wantTTL) - } -} - -func writePromptTestFile(t *testing.T, path string, content string) { - t.Helper() - if err := os.WriteFile(path, []byte(strings.TrimLeft(content, "\n")), 0o644); err != nil { - t.Fatalf("failed to write prompt test file %q: %v", path, err) - } -} - -func copyTree(src, dst string) error { - return filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error { - if err != nil { - return err - } - rel, err := filepath.Rel(src, path) - if err != nil { - return err - } - if rel == "." { - return nil - } - - target := filepath.Join(dst, rel) - if d.IsDir() { - return os.MkdirAll(target, 0o755) - } - - data, err := os.ReadFile(path) - if err != nil { - return err - } - return os.WriteFile(target, data, 0o644) - }) -} diff --git a/internal/promptdef/source.go b/internal/promptdef/source.go deleted file mode 100644 index d8fcbb6..0000000 --- a/internal/promptdef/source.go +++ /dev/null @@ -1,63 +0,0 @@ -package promptdef - -import ( - "context" - "errors" - "io/fs" - "os" - - "gitea.maximumdirect.net/eric/promptkit/internal/filecatalog" -) - -type promptDefinitionSource interface { - contentSourceRoot - findYAMLFiles(context.Context) ([]string, error) - readDefinition(string) ([]byte, error) - displayPath(string) string -} - -type osPromptSource struct { - root string - contentRoot osContentSourceRoot -} - -func (s osPromptSource) findYAMLFiles(ctx context.Context) ([]string, error) { - return filecatalog.FindYAMLFiles(ctx, s.root) -} - -func (s osPromptSource) readDefinition(name string) ([]byte, error) { - return os.ReadFile(name) -} - -func (s osPromptSource) displayPath(name string) string { - return filecatalog.RelativePath(s.root, name) -} - -func (s osPromptSource) readContentFile(sourcePath string, contentFile string) (string, string, error) { - return s.contentRoot.readContentFile(sourcePath, contentFile) -} - -type fsPromptSource struct { - fsys fs.FS - root string - contentRoot contentSourceRoot -} - -func (s fsPromptSource) findYAMLFiles(ctx context.Context) ([]string, error) { - if s.fsys == nil { - return nil, errors.New("filesystem is nil") - } - return filecatalog.FindFSYAMLFiles(ctx, s.fsys, s.root) -} - -func (s fsPromptSource) readDefinition(name string) ([]byte, error) { - return fs.ReadFile(s.fsys, name) -} - -func (s fsPromptSource) displayPath(name string) string { - return filecatalog.DisplayPath(s.root, name) -} - -func (s fsPromptSource) readContentFile(sourcePath string, contentFile string) (string, string, error) { - return s.contentRoot.readContentFile(sourcePath, contentFile) -} diff --git a/internal/promptdef/testdata/both_content_and_content_file.yaml b/internal/promptdef/testdata/both_content_and_content_file.yaml deleted file mode 100644 index de11155..0000000 --- a/internal/promptdef/testdata/both_content_and_content_file.yaml +++ /dev/null @@ -1,10 +0,0 @@ -id: both-content-and-content-file -version: "1.0.0" -messages: - - role: user - content: "Hi" - content_file: ./messages/user_prompt.tmpl -output: - format: text - validation_mode: none - repair_attempts: 0 diff --git a/internal/promptdef/testdata/duplicate_input_names.yaml b/internal/promptdef/testdata/duplicate_input_names.yaml deleted file mode 100644 index 9412e4d..0000000 --- a/internal/promptdef/testdata/duplicate_input_names.yaml +++ /dev/null @@ -1,14 +0,0 @@ -id: duplicate-input-names -version: "1.0.0" -inputs: - - name: transcript - required: true - - name: transcript - required: false -messages: - - role: user - content: "Hi" -output: - format: text - validation_mode: none - repair_attempts: 0 diff --git a/internal/promptdef/testdata/empty_cache_control_type.yaml b/internal/promptdef/testdata/empty_cache_control_type.yaml deleted file mode 100644 index 03330ec..0000000 --- a/internal/promptdef/testdata/empty_cache_control_type.yaml +++ /dev/null @@ -1,10 +0,0 @@ -id: empty-cache-control-type -version: "1.0.0" -messages: - - role: system - content: "Use cached instructions." - cache_control: {} -output: - format: markdown - validation_mode: basic - repair_attempts: 0 diff --git a/internal/promptdef/testdata/invalid_validation_mode.yaml b/internal/promptdef/testdata/invalid_validation_mode.yaml deleted file mode 100644 index 478abc5..0000000 --- a/internal/promptdef/testdata/invalid_validation_mode.yaml +++ /dev/null @@ -1,9 +0,0 @@ -id: invalid-validation-mode -version: "1.0.0" -messages: - - role: user - content: "Hi" -output: - format: text - validation_mode: nope - repair_attempts: 0 diff --git a/internal/promptdef/testdata/invalid_yaml.yaml b/internal/promptdef/testdata/invalid_yaml.yaml deleted file mode 100644 index 8cc64e8..0000000 --- a/internal/promptdef/testdata/invalid_yaml.yaml +++ /dev/null @@ -1,9 +0,0 @@ -id: invalid-yaml -version: "1.0.0" -messages: - - role: user - content: [broken -output: - format: text - validation_mode: none - repair_attempts: 0 diff --git a/internal/promptdef/testdata/json_schema_without_schema_path.yaml b/internal/promptdef/testdata/json_schema_without_schema_path.yaml deleted file mode 100644 index 6dbea5b..0000000 --- a/internal/promptdef/testdata/json_schema_without_schema_path.yaml +++ /dev/null @@ -1,9 +0,0 @@ -id: json-schema-without-schema-path -version: "1.0.0" -messages: - - role: user - content: "Return JSON" -output: - format: json - validation_mode: json_schema - repair_attempts: 0 diff --git a/internal/promptdef/testdata/messages/user_prompt.tmpl b/internal/promptdef/testdata/messages/user_prompt.tmpl deleted file mode 100644 index c8fc459..0000000 --- a/internal/promptdef/testdata/messages/user_prompt.tmpl +++ /dev/null @@ -1,2 +0,0 @@ -Use transcript: -{{input "transcript"}} diff --git a/internal/promptdef/testdata/missing_content_file.yaml b/internal/promptdef/testdata/missing_content_file.yaml deleted file mode 100644 index 42e44fa..0000000 --- a/internal/promptdef/testdata/missing_content_file.yaml +++ /dev/null @@ -1,9 +0,0 @@ -id: missing-content-file -version: "1.0.0" -messages: - - role: user - content_file: ./messages/does_not_exist.tmpl -output: - format: text - validation_mode: none - repair_attempts: 0 diff --git a/internal/promptdef/testdata/missing_id.yaml b/internal/promptdef/testdata/missing_id.yaml deleted file mode 100644 index 6d21ccc..0000000 --- a/internal/promptdef/testdata/missing_id.yaml +++ /dev/null @@ -1,8 +0,0 @@ -version: "1.0.0" -messages: - - role: user - content: "Hi" -output: - format: text - validation_mode: none - repair_attempts: 0 diff --git a/internal/promptdef/testdata/neither_content_nor_content_file.yaml b/internal/promptdef/testdata/neither_content_nor_content_file.yaml deleted file mode 100644 index 525419b..0000000 --- a/internal/promptdef/testdata/neither_content_nor_content_file.yaml +++ /dev/null @@ -1,8 +0,0 @@ -id: neither-content-nor-content-file -version: "1.0.0" -messages: - - role: user -output: - format: text - validation_mode: none - repair_attempts: 0 diff --git a/internal/promptdef/testdata/no_messages.yaml b/internal/promptdef/testdata/no_messages.yaml deleted file mode 100644 index b4b2bbf..0000000 --- a/internal/promptdef/testdata/no_messages.yaml +++ /dev/null @@ -1,6 +0,0 @@ -id: no-messages -version: "1.0.0" -output: - format: text - validation_mode: none - repair_attempts: 0 diff --git a/internal/promptdef/testdata/unknown_cache_control_field.yaml b/internal/promptdef/testdata/unknown_cache_control_field.yaml deleted file mode 100644 index ac8193a..0000000 --- a/internal/promptdef/testdata/unknown_cache_control_field.yaml +++ /dev/null @@ -1,12 +0,0 @@ -id: unknown-cache-control-field -version: "1.0.0" -messages: - - role: system - content: "Use cached instructions." - cache_control: - type: ephemeral - unexpected: true -output: - format: markdown - validation_mode: basic - repair_attempts: 0 diff --git a/internal/promptdef/testdata/unknown_input_field.yaml b/internal/promptdef/testdata/unknown_input_field.yaml deleted file mode 100644 index d8dfa35..0000000 --- a/internal/promptdef/testdata/unknown_input_field.yaml +++ /dev/null @@ -1,13 +0,0 @@ -id: unknown-input-field -version: "1.0.0" -inputs: - - name: transcript - required: true - unknown_input_setting: true -messages: - - role: user - content: "Hi" -output: - format: text - validation_mode: none - repair_attempts: 0 diff --git a/internal/promptdef/testdata/unsupported_cache_control_ttl.yaml b/internal/promptdef/testdata/unsupported_cache_control_ttl.yaml deleted file mode 100644 index d6f81d4..0000000 --- a/internal/promptdef/testdata/unsupported_cache_control_ttl.yaml +++ /dev/null @@ -1,12 +0,0 @@ -id: unsupported-cache-control-ttl -version: "1.0.0" -messages: - - role: system - content: "Use cached instructions." - cache_control: - type: ephemeral - ttl: 5m -output: - format: markdown - validation_mode: basic - repair_attempts: 0 diff --git a/internal/promptdef/testdata/unsupported_cache_control_type.yaml b/internal/promptdef/testdata/unsupported_cache_control_type.yaml deleted file mode 100644 index 875a9b0..0000000 --- a/internal/promptdef/testdata/unsupported_cache_control_type.yaml +++ /dev/null @@ -1,11 +0,0 @@ -id: unsupported-cache-control-type -version: "1.0.0" -messages: - - role: system - content: "Use cached instructions." - cache_control: - type: persistent -output: - format: markdown - validation_mode: basic - repair_attempts: 0 diff --git a/internal/promptdef/testdata/valid_cache_control_ttl.yaml b/internal/promptdef/testdata/valid_cache_control_ttl.yaml deleted file mode 100644 index 4136ac4..0000000 --- a/internal/promptdef/testdata/valid_cache_control_ttl.yaml +++ /dev/null @@ -1,14 +0,0 @@ -id: valid-cache-control-ttl -version: "1.0.0" -messages: - - role: system - content: "Use cached instructions." - cache_control: - type: ephemeral - ttl: 1h - - role: user - content: "Summarize the input." -output: - format: markdown - validation_mode: basic - repair_attempts: 0 diff --git a/internal/promptdef/testdata/valid_cache_control_without_ttl.yaml b/internal/promptdef/testdata/valid_cache_control_without_ttl.yaml deleted file mode 100644 index 48e6c37..0000000 --- a/internal/promptdef/testdata/valid_cache_control_without_ttl.yaml +++ /dev/null @@ -1,13 +0,0 @@ -id: valid-cache-control-without-ttl -version: "1.0.0" -messages: - - role: system - content: "Use cached instructions." - cache_control: - type: ephemeral - - role: user - content: "Summarize the input." -output: - format: markdown - validation_mode: basic - repair_attempts: 0 diff --git a/internal/promptdef/testdata/valid_file_backed.yaml b/internal/promptdef/testdata/valid_file_backed.yaml deleted file mode 100644 index f57f56a..0000000 --- a/internal/promptdef/testdata/valid_file_backed.yaml +++ /dev/null @@ -1,14 +0,0 @@ -id: valid-file-backed -version: "1.0.0" -inputs: - - name: transcript - required: true -messages: - - role: system - content: "Return markdown." - - role: user - content_file: ./messages/user_prompt.tmpl -output: - format: markdown - validation_mode: basic - repair_attempts: 0 diff --git a/internal/promptdef/testdata/valid_inline.yaml b/internal/promptdef/testdata/valid_inline.yaml deleted file mode 100644 index 3f61059..0000000 --- a/internal/promptdef/testdata/valid_inline.yaml +++ /dev/null @@ -1,18 +0,0 @@ -id: valid-inline -version: "1.0.0" -inputs: - - name: transcript - required: true - content_type: text/markdown - description: Transcript content -messages: - - role: system - content: "You are concise." - - role: user - content: | - Summarize: - {{input "transcript"}} -output: - format: markdown - validation_mode: basic - repair_attempts: 0 diff --git a/internal/promptdef/testdata/valid_session_id.yaml b/internal/promptdef/testdata/valid_session_id.yaml deleted file mode 100644 index b477662..0000000 --- a/internal/promptdef/testdata/valid_session_id.yaml +++ /dev/null @@ -1,10 +0,0 @@ -id: valid-session-id -version: "1.0.0" -session_id: " {{ .session_id }} " -messages: - - role: user - content: Hello. -output: - format: markdown - validation_mode: basic - repair_attempts: 0 diff --git a/internal/promptdef/testdata/with_default_profile.yaml b/internal/promptdef/testdata/with_default_profile.yaml deleted file mode 100644 index 1628b41..0000000 --- a/internal/promptdef/testdata/with_default_profile.yaml +++ /dev/null @@ -1,13 +0,0 @@ -id: with-default-profile -version: "1.0.0" -default_profile: local-default -inputs: - - name: transcript - required: true -messages: - - role: user - content: "Write output" -output: - format: text - validation_mode: none - repair_attempts: 0 diff --git a/internal/usecase/execution_settings_test.go b/internal/usecase/execution_settings_test.go index 912d046..8cfff65 100644 --- a/internal/usecase/execution_settings_test.go +++ b/internal/usecase/execution_settings_test.go @@ -13,7 +13,7 @@ func TestRunnerPrepareExecutionRejectsInvalidExecutionSettings(t *testing.T) { runner := NewRunner( &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, - nil, + defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, @@ -37,7 +37,7 @@ func TestRunnerPrepareExecutionValidatesAndNormalizesRequestEndpoints(t *testing return NewRunner( &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, - nil, + defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, diff --git a/internal/usecase/output_contract_test.go b/internal/usecase/output_contract_test.go index a6e5873..81a3aed 100644 --- a/internal/usecase/output_contract_test.go +++ b/internal/usecase/output_contract_test.go @@ -27,7 +27,7 @@ func newOutputContractTestRunner() (*Runner, outputContractTestCollaborators) { return NewRunner( &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, - nil, + collaborators.artifacts, collaborators.renderer, collaborators.llm, diff --git a/internal/usecase/prepared_execution_test.go b/internal/usecase/prepared_execution_test.go index dd2d3f3..ab968be 100644 --- a/internal/usecase/prepared_execution_test.go +++ b/internal/usecase/prepared_execution_test.go @@ -104,13 +104,13 @@ func TestRunnerPrepareExecutionCompletesWithoutAdmissionOrGeneration(t *testing. } admitter := &fakeRunAdmitter{} profile := defaultExecutionProfile() - profile.ExtraParams = map[string]any{ + profile.ResolvedTarget.ExtraParams = map[string]any{ "metadata": map[string]any{"source": "original"}, } runner := NewRunner( &fakePromptRepo{def: def}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}}, - nil, + reader, renderer, llmClient, @@ -210,7 +210,7 @@ func TestRunnerPreparationRejectsExcessivelyDeepPreparedSchema(t *testing.T) { runner := NewRunner( &fakePromptRepo{def: def}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, - nil, + defaultArtifactReader(), defaultRenderer(), llmClient, @@ -265,9 +265,9 @@ func TestRunnerRunPreparedCredentialAvailabilityBeforeAdmission(t *testing.T) { t.Setenv(environmentName, "available-during-preparation") profile := defaultExecutionProfile() - profile.APIKeyRequired = tc.apiKeyRequired + profile.ResolvedTarget.APIKeyRequired = tc.apiKeyRequired if tc.profileEnv { - profile.APIKeyEnv = environmentName + profile.ResolvedTarget.APIKeyEnv = environmentName } validator := &recordingValidationPreparer{plan: &recordingPreparedValidation{}} admitter := &fakeRunAdmitter{} @@ -275,7 +275,7 @@ func TestRunnerRunPreparedCredentialAvailabilityBeforeAdmission(t *testing.T) { runner := NewRunner( &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}}, - nil, + defaultArtifactReader(), defaultRenderer(), llmClient, @@ -319,12 +319,12 @@ func TestRunnerRunPreparedKeepsDirectCredentialOutOfMetadata(t *testing.T) { const directKey = "direct-prepared-test-key" profile := defaultExecutionProfile() - profile.APIKeyRequired = true + profile.ResolvedTarget.APIKeyRequired = true client := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}} runner := NewRunner( &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": profile}}, - nil, + defaultArtifactReader(), defaultRenderer(), client, @@ -397,7 +397,7 @@ func TestRunnerRunPreparedUsesFrozenValidationForInitialAndRepairOutputs(t *test runner := NewRunnerWithRepairer( &fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, - nil, + reader, renderer, client, @@ -542,7 +542,7 @@ func TestRunnerRunPreparedReleasesAdmissionAcrossExecutionErrors(t *testing.T) { runner := NewRunnerWithRepairer( &fakePromptRepo{def: def}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, - nil, + defaultArtifactReader(), defaultRenderer(), &fakeLLM{ @@ -585,7 +585,7 @@ func TestRunnerPreparedExecutionOwnershipUseAndDiscard(t *testing.T) { return NewRunner( &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, - nil, + defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, @@ -635,7 +635,7 @@ func TestRunnerPreparedExecutionWithoutValidatorSkipsValidation(t *testing.T) { runner := NewRunner( &fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 0)}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, - nil, + defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: `{}`}}, @@ -665,7 +665,7 @@ func TestRunnerPrepareExecutionRequiresValidationPreparer(t *testing.T) { runner := NewRunner( &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, - nil, + reader, defaultRenderer(), &fakeLLM{forbid: true}, diff --git a/internal/usecase/profile_inspection.go b/internal/usecase/profile_inspection.go index ba74515..4516917 100644 --- a/internal/usecase/profile_inspection.go +++ b/internal/usecase/profile_inspection.go @@ -7,12 +7,12 @@ import ( "strings" "gitea.maximumdirect.net/eric/promptkit/internal/domain" + "gitea.maximumdirect.net/eric/promptkit/internal/jsonvalue" ) type resolvedProfileSelection struct { id string profile *domain.ExecutionProfile - backend *domain.Backend } func (r *Runner) resolveProfileSelection( @@ -26,10 +26,6 @@ func (r *Runner) resolveProfileSelection( if r == nil || r.profiles == nil { return nil, fmt.Errorf("%w: profile repository is not configured", ErrProfileLoad) } - if exact, ok := r.profiles.(interface{ ExactIDs() bool }); !ok || !exact.ExactIDs() { - lookupID = strings.TrimSpace(lookupID) - } - selectedProfile, err := r.profiles.GetProfile(ctx, lookupID) if err != nil { return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err) @@ -39,27 +35,13 @@ func (r *Runner) resolveProfileSelection( } profileValue := *selectedProfile - if profileValue.ResolvedTarget != nil { - return &resolvedProfileSelection{id: lookupID, profile: &profileValue}, nil - } - profileValue.BackendID = strings.TrimSpace(profileValue.BackendID) - - var selectedBackend *domain.Backend - if profileValue.BackendID != "" { - if r.backends == nil { - return nil, fmt.Errorf("%w: backend %q cannot be resolved", ErrProfileLoad, profileValue.BackendID) - } - backendValue, err := r.backends.GetBackend(profileValue.BackendID) - if err != nil { - return nil, fmt.Errorf("%w: backend %q: %w", ErrProfileLoad, profileValue.BackendID, err) - } - selectedBackend = &backendValue + if profileValue.ResolvedTarget == nil { + return nil, fmt.Errorf("%w: profile %q has no resolved target", ErrProfileLoad, lookupID) } return &resolvedProfileSelection{ id: lookupID, profile: &profileValue, - backend: selectedBackend, }, nil } @@ -96,14 +78,13 @@ func (r *Runner) InspectProfile( if err != nil { return nil, err } - target, _ := resolveExecutionTarget(selection.backend, selection.profile, nil) - if selection.profile.ResolvedTarget == nil { - target, err = normalizeResolvedExecutionTarget(target) - if err != nil { - return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err) - } - } + target, _ := resolveExecutionTarget(selection.profile, nil) target.APIKey = "" + params, err := jsonvalue.CopyMap(target.ExtraParams) + if err != nil { + return nil, fmt.Errorf("%w: profile extra parameters: %w", ErrProfileLoad, err) + } + target.ExtraParams = params return &domain.ProfileInspection{ ProfileID: selection.id, diff --git a/internal/usecase/profile_inspection_test.go b/internal/usecase/profile_inspection_test.go index c179006..b4a7839 100644 --- a/internal/usecase/profile_inspection_test.go +++ b/internal/usecase/profile_inspection_test.go @@ -6,7 +6,6 @@ import ( "reflect" "testing" - "gitea.maximumdirect.net/eric/promptkit/internal/defaults" "gitea.maximumdirect.net/eric/promptkit/internal/domain" "gitea.maximumdirect.net/eric/promptkit/internal/profile" ) @@ -18,10 +17,7 @@ type inspectionProfileRepository struct { id string } -func (r *inspectionProfileRepository) GetProfile( - _ context.Context, - id string, -) (*domain.ExecutionProfile, error) { +func (r *inspectionProfileRepository) GetProfile(_ context.Context, id string) (*domain.ExecutionProfile, error) { r.calls++ r.id = id if r.err != nil { @@ -30,184 +26,65 @@ func (r *inspectionProfileRepository) GetProfile( return r.profile, nil } -type inspectionBackendResolver struct { - backend domain.Backend - err error - calls int - id string -} - -func (r *inspectionBackendResolver) GetBackend(id string) (domain.Backend, error) { - r.calls++ - r.id = id - if r.err != nil { - return domain.Backend{}, r.err +func TestRunnerInspectProfileUsesExactSnapshotAndCopiesTarget(t *testing.T) { + configured := domain.ExecutionTarget{ + BackendID: "backend", Endpoint: "https://backend.example/v1", Model: "model", + Temperature: 0, APIKeyEnv: "OPTIONAL_KEY", ExtraParams: map[string]any{"nested": map[string]any{"value": "original"}}, } - return r.backend, nil -} - -func TestRunnerInspectProfileResolvesProfileAndBackendOnce(t *testing.T) { profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{ - ID: "profile", - BackendID: " backend ", - Model: "profile-model", - Temperature: 0.4, - MaxTokens: 32, - TimeoutSeconds: 45, - ServiceTier: "priority", - ReasoningEffort: "high", - ExtraParams: map[string]any{ - "profile": "value", - }, + ID: " literal ", ResolvedTarget: &configured, + TargetPresence: domain.ExecutionTargetPresence{Temperature: true}, }} - backends := &inspectionBackendResolver{backend: domain.Backend{ - ID: "backend", - Endpoint: "https://backend.example/v1", - APIKeyEnv: "BACKEND_KEY", - ExtraParams: map[string]any{ - "backend": "value", - }, - }} - runner := &Runner{profiles: profiles, backends: backends} - - inspection, err := runner.InspectProfile(context.Background(), " profile ") + runner := &Runner{profiles: profiles} + first, err := runner.InspectProfile(context.Background(), " literal ") if err != nil { - t.Fatalf("inspect profile: %v", err) + t.Fatalf("inspect exact ID: %v", err) } - if profiles.calls != 1 || profiles.id != "profile" { - t.Fatalf("profile lookup=(calls=%d id=%q), want one exact lookup", profiles.calls, profiles.id) + if profiles.calls != 1 || profiles.id != " literal " || first.ProfileID != profiles.id || !reflect.DeepEqual(first.EffectiveModelParams, configured) { + t.Fatalf("inspection=%+v lookup=(%d,%q), want exact snapshot", first, profiles.calls, profiles.id) } - if backends.calls != 1 || backends.id != "backend" { - t.Fatalf("backend lookup=(calls=%d id=%q), want one exact lookup", backends.calls, backends.id) - } - if profiles.profile.BackendID != " backend " { - t.Fatalf("inspection mutated repository profile backend: %q", profiles.profile.BackendID) - } - - wantTarget := domain.ExecutionTarget{ - BackendID: "backend", - Endpoint: "https://backend.example/v1", - Model: "profile-model", - Temperature: 0.4, - MaxTokens: 32, - TopP: defaults.ExecutionTargetDefault().TopP, - TimeoutSeconds: 45, - ServiceTier: "priority", - ReasoningEffort: "high", - APIKeyEnv: "BACKEND_KEY", - ExtraParams: map[string]any{ - "profile": "value", - }, - } - if inspection.ProfileID != "profile" || inspection.APIKeyRequired || - !reflect.DeepEqual(inspection.EffectiveModelParams, wantTarget) { - t.Fatalf("inspection=%#v, want profile=%q target=%#v", inspection, "profile", wantTarget) + first.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["value"] = "changed" + second, err := runner.InspectProfile(context.Background(), " literal ") + if err != nil || second.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["value"] != "original" { + t.Fatalf("inspection mutated snapshot: (%+v, %v)", second, err) } } func TestRunnerInspectProfileDoesNotNeedExecutionCollaboratorsOrCredentials(t *testing.T) { t.Setenv("PROMPTKIT_INSPECTION_TEST_KEY", "") profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{ - ID: "endpoint-only", - Endpoint: "https://profile.example/v1", - Model: "profile-model", - APIKeyEnv: "PROMPTKIT_INSPECTION_TEST_KEY", + ID: "endpoint-only", ResolvedTarget: &domain.ExecutionTarget{ + Endpoint: "https://profile.example/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_INSPECTION_TEST_KEY", + }, }} + inspection, err := (&Runner{profiles: profiles}).InspectProfile(context.Background(), "endpoint-only") + if err != nil || inspection.EffectiveModelParams.APIKeyEnv != "PROMPTKIT_INSPECTION_TEST_KEY" || inspection.APIKeyRequired { + t.Fatalf("unexpected endpoint-only inspection: (%+v, %v)", inspection, err) + } +} + +func TestRunnerInspectProfileClassifiesLookupFailures(t *testing.T) { + profiles := &inspectionProfileRepository{} runner := &Runner{profiles: profiles} - - inspection, err := runner.InspectProfile(context.Background(), "endpoint-only") - if err != nil { - t.Fatalf("inspect endpoint-only profile: %v", err) + if _, err := runner.InspectProfile(context.Background(), " \t "); !errors.Is(err, ErrInvalidRequest) || profiles.calls != 0 { + t.Fatalf("blank inspection=(%v, calls=%d)", err, profiles.calls) } - if inspection.EffectiveModelParams.BackendID != "" || - inspection.EffectiveModelParams.APIKeyEnv != "PROMPTKIT_INSPECTION_TEST_KEY" || - inspection.APIKeyRequired { - t.Fatalf("unexpected endpoint-only inspection: %#v", inspection) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := runner.InspectProfile(ctx, "profile"); !errors.Is(err, ErrProfileLoad) || !errors.Is(err, context.Canceled) || profiles.calls != 0 { + t.Fatalf("canceled inspection=(%v, calls=%d)", err, profiles.calls) + } + profiles.err = profile.ErrProfileNotFound + if _, err := runner.InspectProfile(context.Background(), "missing"); !errors.Is(err, ErrProfileLoad) || !errors.Is(err, profile.ErrProfileNotFound) { + t.Fatalf("missing profile error=%v", err) + } + for _, invalid := range []*Runner{ + {}, + {profiles: &inspectionProfileRepository{}}, + {profiles: &inspectionProfileRepository{profile: &domain.ExecutionProfile{ID: "profile"}}}, + } { + if _, err := invalid.InspectProfile(context.Background(), "profile"); !errors.Is(err, ErrProfileLoad) { + t.Fatalf("invalid dependency error=%v", err) + } } } - -func TestRunnerInspectProfileDirectCredentialRequirementClearsBackendEnvironment(t *testing.T) { - profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{ - ID: "direct-key", - BackendID: "backend", - Model: "profile-model", - APIKeyRequired: true, - }} - backends := &inspectionBackendResolver{backend: domain.Backend{ - ID: "backend", - Endpoint: "https://backend.example/v1", - APIKeyEnv: "BACKEND_KEY", - }} - - inspection, err := (&Runner{profiles: profiles, backends: backends}).InspectProfile( - context.Background(), - "direct-key", - ) - if err != nil { - t.Fatalf("inspect direct-key profile: %v", err) - } - if !inspection.APIKeyRequired || inspection.EffectiveModelParams.APIKeyEnv != "" { - t.Fatalf("credential requirement was not resolved exclusively: %#v", inspection) - } -} - -func TestRunnerInspectProfileClassifiesFailuresWithoutRepositoryWorkAfterCancellation(t *testing.T) { - t.Run("blank ID", func(t *testing.T) { - profiles := &inspectionProfileRepository{} - _, err := (&Runner{profiles: profiles}).InspectProfile(context.Background(), " \t ") - if !errors.Is(err, ErrInvalidRequest) || profiles.calls != 0 { - t.Fatalf("blank inspection=(%v, calls=%d), want invalid request without lookup", err, profiles.calls) - } - }) - - t.Run("canceled context", func(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - cancel() - profiles := &inspectionProfileRepository{} - _, err := (&Runner{profiles: profiles}).InspectProfile(ctx, "profile") - if !errors.Is(err, ErrProfileLoad) || !errors.Is(err, context.Canceled) || profiles.calls != 0 { - t.Fatalf("canceled inspection=(%v, calls=%d), want profile load and context identities without lookup", err, profiles.calls) - } - }) - - t.Run("missing profile", func(t *testing.T) { - profiles := &inspectionProfileRepository{err: profile.ErrProfileNotFound} - _, err := (&Runner{profiles: profiles}).InspectProfile(context.Background(), "missing") - if !errors.Is(err, ErrProfileLoad) || !errors.Is(err, profile.ErrProfileNotFound) { - t.Fatalf("missing profile error=%v, want profile load and not-found identities", err) - } - }) - - t.Run("unknown backend", func(t *testing.T) { - backendErr := errors.New("unknown backend") - profiles := &inspectionProfileRepository{profile: &domain.ExecutionProfile{ - ID: "profile", BackendID: "backend", Model: "profile-model", - }} - backends := &inspectionBackendResolver{err: backendErr} - _, err := (&Runner{profiles: profiles, backends: backends}).InspectProfile(context.Background(), "profile") - if !errors.Is(err, ErrProfileLoad) || !errors.Is(err, backendErr) { - t.Fatalf("unknown backend error=%v, want profile load and backend identities", err) - } - }) - - t.Run("defensive invalid dependencies", func(t *testing.T) { - cases := []struct { - name string - runner *Runner - }{ - {name: "nil repository", runner: &Runner{}}, - {name: "nil profile", runner: &Runner{profiles: &inspectionProfileRepository{}}}, - {name: "invalid target", runner: &Runner{profiles: &inspectionProfileRepository{ - profile: &domain.ExecutionProfile{ID: "profile", Endpoint: "https://profile.example/v1"}, - }}}, - } - for _, tc := range cases { - t.Run(tc.name, func(t *testing.T) { - _, err := tc.runner.InspectProfile(context.Background(), "profile") - if !errors.Is(err, ErrProfileLoad) { - t.Fatalf("inspection error=%v, want profile load", err) - } - }) - } - }) -} diff --git a/internal/usecase/prompt_inspection_test.go b/internal/usecase/prompt_inspection_test.go index 9560dd0..c26acdc 100644 --- a/internal/usecase/prompt_inspection_test.go +++ b/internal/usecase/prompt_inspection_test.go @@ -135,7 +135,7 @@ func TestRunnerPrepareUsesThePromptInspectionSelectionAndHash(t *testing.T) { runner := NewRunner( repository, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, - nil, + &fakeArtifactReader{}, &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "user", Content: "hello"}}}}, nil, diff --git a/internal/usecase/runner.go b/internal/usecase/runner.go index e3909cb..387d3c3 100644 --- a/internal/usecase/runner.go +++ b/internal/usecase/runner.go @@ -44,7 +44,6 @@ var ( type Runner struct { promptDefs promptdef.Repository profiles profile.Repository - backends BackendResolver artifacts artifact.Reader renderer prompt.Renderer llm llm.Client @@ -54,11 +53,6 @@ type Runner struct { defaultTransientRetries uint64 } -// BackendResolver resolves one normalized backend ID. -type BackendResolver interface { - GetBackend(string) (domain.Backend, error) -} - // RunAdmitter reserves capacity for one resolved backend run. type RunAdmitter interface { Admit(context.Context, string) (func(), error) @@ -84,7 +78,6 @@ type preparedOperation struct { func NewRunner( promptDefs promptdef.Repository, profiles profile.Repository, - backends BackendResolver, artifacts artifact.Reader, renderer prompt.Renderer, llmClient llm.Client, @@ -94,7 +87,6 @@ func NewRunner( return NewRunnerWithRepairer( promptDefs, profiles, - backends, artifacts, renderer, llmClient, @@ -107,7 +99,6 @@ func NewRunner( func NewRunnerWithRepairer( promptDefs promptdef.Repository, profiles profile.Repository, - backends BackendResolver, artifacts artifact.Reader, renderer prompt.Renderer, llmClient llm.Client, @@ -123,7 +114,6 @@ func NewRunnerWithRepairer( return &Runner{ promptDefs: promptDefs, profiles: profiles, - backends: backends, artifacts: artifacts, renderer: renderer, llm: llmClient, @@ -355,7 +345,7 @@ func (r *Runner) resolvePreparation( recorder.UpdateSelection(observation.Selection{PromptID: def.ID, PromptVersion: def.Version, ProfileID: selection.id}) } - effectiveModel, targetPresence := resolveExecutionTarget(selection.backend, selection.profile, req.Execution) + effectiveModel, targetPresence := resolveExecutionTarget(selection.profile, req.Execution) effectiveModel.APIKey = req.APIKey effectiveModel, err = normalizeResolvedExecutionTarget(effectiveModel) if err != nil { @@ -578,48 +568,6 @@ func (r *Runner) shouldAttemptRepair(contract domain.OutputContract, validationR } } -func mergeExecutionTarget(base domain.ExecutionTarget, override domain.ExecutionTarget) domain.ExecutionTarget { - out := base - if strings.TrimSpace(override.BackendID) != "" { - out.BackendID = override.BackendID - } - if strings.TrimSpace(override.Endpoint) != "" { - out.Endpoint = override.Endpoint - } - if override.Model != "" { - out.Model = override.Model - } - if override.Temperature != 0 { - out.Temperature = override.Temperature - } - if override.MaxTokens != 0 { - out.MaxTokens = override.MaxTokens - } - if override.TopP != 0 { - out.TopP = override.TopP - } - if override.TimeoutSeconds != 0 { - out.TimeoutSeconds = override.TimeoutSeconds - } - if strings.TrimSpace(override.ServiceTier) != "" { - out.ServiceTier = override.ServiceTier - } - if strings.TrimSpace(override.ReasoningEffort) != "" { - out.ReasoningEffort = override.ReasoningEffort - } - if strings.TrimSpace(override.APIKeyEnv) != "" { - out.APIKeyEnv = override.APIKeyEnv - } - if override.APIKeyRequired { - out.APIKeyRequired = true - out.APIKeyEnv = "" - } - if len(override.ExtraParams) > 0 { - out.ExtraParams = copyExtraParams(override.ExtraParams) - } - return out -} - func mergeExecutionTargetOverride(base domain.ExecutionTarget, presence domain.ExecutionTargetPresence, override domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence) { out := base if override.Endpoint != "" { @@ -659,22 +607,12 @@ func mergeExecutionTargetOverride(base domain.ExecutionTarget, presence domain.E return out, presence } -func resolveExecutionTarget(backendValue *domain.Backend, profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence) { - if profileValue != nil && profileValue.ResolvedTarget != nil { - out := *profileValue.ResolvedTarget - out.ExtraParams = copyExtraParams(out.ExtraParams) - presence := profileValue.TargetPresence - if override != nil { - return mergeExecutionTargetOverride(out, presence, *override) - } - return out, presence - } - out := defaults.ExecutionTargetDefault() - out = mergeExecutionTarget(out, backendToTarget(backendValue)) - out = mergeExecutionTarget(out, executionProfileToTarget(profileValue)) - var presence domain.ExecutionTargetPresence +func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence) { + out := *profileValue.ResolvedTarget + out.ExtraParams = copyExtraParams(out.ExtraParams) + presence := profileValue.TargetPresence if override != nil { - out, presence = mergeExecutionTargetOverride(out, presence, *override) + return mergeExecutionTargetOverride(out, presence, *override) } return out, presence } @@ -696,42 +634,6 @@ func validateAPIKey(apiKeyEnv string, apiKey string, apiKeyRequired bool) error return nil } -func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget { - if p == nil { - return domain.ExecutionTarget{} - } - endpoint := p.Endpoint - if strings.TrimSpace(endpoint) == "" { - endpoint = "" - } - return domain.ExecutionTarget{ - BackendID: p.BackendID, - Endpoint: endpoint, - Model: p.Model, - Temperature: p.Temperature, - MaxTokens: p.MaxTokens, - TopP: p.TopP, - TimeoutSeconds: p.TimeoutSeconds, - ServiceTier: p.ServiceTier, - ReasoningEffort: p.ReasoningEffort, - APIKeyEnv: p.APIKeyEnv, - APIKeyRequired: p.APIKeyRequired, - ExtraParams: copyExtraParams(p.ExtraParams), - } -} - -func backendToTarget(value *domain.Backend) domain.ExecutionTarget { - if value == nil { - return domain.ExecutionTarget{} - } - return domain.ExecutionTarget{ - BackendID: value.ID, - Endpoint: value.Endpoint, - APIKeyEnv: value.APIKeyEnv, - ExtraParams: copyExtraParams(value.ExtraParams), - } -} - func copyExtraParams(src map[string]any) map[string]any { if len(src) == 0 { return nil diff --git a/internal/usecase/runner_test.go b/internal/usecase/runner_test.go index 86a203b..7bb4540 100644 --- a/internal/usecase/runner_test.go +++ b/internal/usecase/runner_test.go @@ -7,7 +7,6 @@ import ( "errors" "fmt" "math" - "path/filepath" "reflect" "regexp" "strconv" @@ -20,9 +19,7 @@ import ( "gitea.maximumdirect.net/eric/promptkit/internal/defaults" "gitea.maximumdirect.net/eric/promptkit/internal/domain" "gitea.maximumdirect.net/eric/promptkit/internal/llm" - "gitea.maximumdirect.net/eric/promptkit/internal/profile" "gitea.maximumdirect.net/eric/promptkit/internal/prompt" - "gitea.maximumdirect.net/eric/promptkit/internal/promptdef" "gitea.maximumdirect.net/eric/promptkit/internal/validate" ) @@ -39,10 +36,6 @@ type fakeExecutionProfileRepo struct { lastID string } -type fakeBackendResolver struct { - backends map[string]domain.Backend -} - type staticPromptRepo struct { def *domain.PromptDefinition } @@ -89,14 +82,6 @@ func (staticRenderer) Render( }, nil } -func (f fakeBackendResolver) GetBackend(id string) (domain.Backend, error) { - value, ok := f.backends[id] - if !ok { - return domain.Backend{}, errors.New("backend not found") - } - return value, nil -} - func (f *fakeExecutionProfileRepo) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) { f.lastID = id if f.err != nil { @@ -371,7 +356,7 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) { renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}} llmClient := &fakeLLM{forbid: true} - runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil, nil) + runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil, nil) prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ PromptID: "p", PromptVersion: "1", @@ -422,7 +407,7 @@ func TestRunnerDirectSessionResolution(t *testing.T) { runner := NewRunner( promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, - nil, + defaultArtifactReader(), prompt.NewGoRenderer(), &fakeLLM{forbid: true}, @@ -469,7 +454,7 @@ func TestRunnerDirectSessionResolution(t *testing.T) { runner := NewRunner( &fakePromptRepo{def: def}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, - nil, + defaultArtifactReader(), prompt.NewGoRenderer(), &fakeLLM{forbid: true}, @@ -499,7 +484,7 @@ func TestRunnerDirectSessionResolution(t *testing.T) { runner := NewRunner( &fakePromptRepo{def: def}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, - nil, + defaultArtifactReader(), prompt.NewGoRenderer(), &fakeLLM{forbid: true}, @@ -526,7 +511,7 @@ func TestRunnerDirectSessionResolution(t *testing.T) { runner := NewRunner( promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, - nil, + defaultArtifactReader(), defaultRenderer(), llmClient, @@ -555,7 +540,7 @@ func TestRunnerDirectSessionResolution(t *testing.T) { runner := NewRunner( promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, - nil, + defaultArtifactReader(), defaultRenderer(), llmClient, @@ -583,7 +568,7 @@ func TestRunnerPrepareUsesPromptDefaultProfileWhenNoExplicitProfileID(t *testing promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} promptRepo.def.DefaultProfile = "from-prompt" execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "from-prompt": {ID: "from-prompt", Endpoint: "http://llm/v1", Model: "m"}, + "from-prompt": {ID: "from-prompt", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m", TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds}}, }} runner := newMinimalRunner(promptRepo, execRepo) @@ -626,7 +611,7 @@ func TestRunnerPrepareSelectedProfileDoesNotExistFails(t *testing.T) { } func TestRunnerPreparePromptLoadFailure(t *testing.T) { - runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil, nil) + runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil, nil) _, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p"}) if !errors.Is(err, ErrPromptLoad) { t.Fatalf("expected ErrPromptLoad, got %v", err) @@ -640,17 +625,16 @@ func TestRunnerPrepareRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) { promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ "exec": { - ID: "exec", - Endpoint: "http://profile/v1", - Model: "profile-model", - Temperature: 0.2, - MaxTokens: 500, - TopP: 0.9, - TimeoutSeconds: 120, - ServiceTier: "priority", + ID: "exec", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://profile/v1", + Model: "profile-model", + Temperature: 0.2, + MaxTokens: 500, + TopP: 0.9, + TimeoutSeconds: 120, + ServiceTier: "priority"}, }, }} - runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil, nil) + runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil, nil) prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ PromptID: "p", @@ -740,15 +724,14 @@ func TestRunnerPrepareRequestNumericOverridePresence(t *testing.T) { &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ "exec": { - ID: "exec", - Endpoint: "http://profile/v1", - Model: "profile-model", - Temperature: 0.7, - MaxTokens: 321, - TopP: 0.8, - TimeoutSeconds: 45, + ID: "exec", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://profile/v1", + Model: "profile-model", + Temperature: 0.7, + MaxTokens: 321, + TopP: 0.8, + TimeoutSeconds: 45}, }, - }}, nil, + }}, defaultArtifactReader(), defaultRenderer(), @@ -802,7 +785,7 @@ func TestRunnerPrepareInvalidRequestNumericOverridesFail(t *testing.T) { t.Run(tc.name, func(t *testing.T) { runner := NewRunner( &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), @@ -826,15 +809,14 @@ func TestRunnerPrepareSelectedProfileBeatsBuiltInDefault(t *testing.T) { promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ "exec": { - ID: "exec", - Endpoint: "http://profile/v1", - Model: "profile-model", - TopP: 0.8, - TimeoutSeconds: 90, - ServiceTier: "priority", + ID: "exec", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://profile/v1", + Model: "profile-model", + TopP: 0.8, + TimeoutSeconds: 90, + ServiceTier: "priority"}, }, }} - runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil, nil) + runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil, nil) prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ PromptID: "p", @@ -855,54 +837,13 @@ func TestRunnerPrepareSelectedProfileBeatsBuiltInDefault(t *testing.T) { } } -func TestRunnerPrepareFileBackedPromptBodiesRenderCorrectly(t *testing.T) { - promptDir := filepath.Join("..", "promptdef", "testdata") - profileDir := filepath.Join("..", "profile", "testdata") - - reader := &fakeArtifactReader{ - artifactsByURI: map[string]*domain.Artifact{ - "a://transcript": { - Name: "transcript", - Body: []byte("Session transcript body."), - Hash: hashString("Session transcript body."), - }, - }, - } - llmClient := &fakeLLM{forbid: true} - runner := NewRunner( - promptdef.NewFilesystemRepository(promptDir), - profile.NewFilesystemRepository(profileDir), nil, - - reader, - prompt.NewGoRenderer(), - llmClient, - nil, nil) - - prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ - PromptID: "valid-file-backed", - ProfileID: "local-default", - Inputs: map[string]domain.ArtifactRef{ - "transcript": {Type: domain.ArtifactRefFile, URI: "a://transcript"}, - }, - }) - if err != nil { - t.Fatalf("expected no error, got %v", err) - } - if len(prepared.Messages) != 2 { - t.Fatalf("expected two rendered messages, got %d", len(prepared.Messages)) - } - if !strings.Contains(prepared.Messages[1].Content, "Session transcript body.") { - t.Fatalf("expected file-backed template content to render input, got %q", prepared.Messages[1].Content) - } -} - func TestRunnerPrepareRequiredInputMissingFails(t *testing.T) { def := promptDef(domain.FormatText, domain.ValidationNone, 0) def.Templates = []domain.PromptMessageTemplate{{Role: "user", Content: `{{input "transcript"}}`}} runner := NewRunner( &fakePromptRepo{def: def}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), prompt.NewGoRenderer(), @@ -929,7 +870,7 @@ func TestRunnerPrepareUnknownTemplateInputReferenceFails(t *testing.T) { runner := NewRunner( &fakePromptRepo{def: def}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), prompt.NewGoRenderer(), @@ -955,9 +896,9 @@ func TestRunnerPrepareAPIKeyEnvNameIncludedButNotResolvedValue(t *testing.T) { t.Setenv(envName, secret) promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName}, + "exec": {ID: "exec", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName, TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds}}, }} - runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil, nil) + runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil, nil) prepared, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()}) if err != nil { @@ -985,7 +926,7 @@ func TestRunnerPrepareJSONSchemaBuildsStructuredOutputSpec(t *testing.T) { } runner := NewRunner( &fakePromptRepo{def: def}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), @@ -1029,7 +970,7 @@ func TestRunnerPrepareJSONSchemaSchemaLoadFailureReturnsValidationError(t *testi validator := &fakeValidator{schemaErr: errors.New("schema unavailable")} runner := NewRunner( &fakePromptRepo{def: def}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), @@ -1059,7 +1000,7 @@ func TestRunnerRunJSONSchemaSchemaLoadFailureFailsBeforeLLM(t *testing.T) { validator := &fakeValidator{schemaErr: errors.New("schema unavailable")} runner := NewRunner( &fakePromptRepo{def: def}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), @@ -1243,7 +1184,7 @@ func TestRunnerComposesAppendedMessagesIntoEveryRenderedPrompt(t *testing.T) { runner := NewRunner( &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, - nil, + defaultArtifactReader(), renderer, client, @@ -1310,7 +1251,7 @@ func TestRunnerRunSuccessful(t *testing.T) { renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}} - runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil, nil) + runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil, nil) res, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", PromptVersion: "1", @@ -1385,14 +1326,13 @@ func TestRunnerRunPassesExtraParamsToGenerateRequestTarget(t *testing.T) { promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ "exec": { - ID: "exec", - Endpoint: "http://profile/v1", - Model: "profile-model", - ExtraParams: extraParams, + ID: "exec", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://profile/v1", + Model: "profile-model", + ExtraParams: extraParams, TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds}, }, }} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}} - runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil) + runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil) res, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", @@ -1418,7 +1358,7 @@ func TestRunnerRunAndPrepareResolveSameProfileAndEffectiveSettings(t *testing.T) }} renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap"}} - runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil, nil) + runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil, nil) req := domain.RunRequest{ PromptID: "p", @@ -1463,14 +1403,11 @@ func TestRunnerAdmissionUsesResolvedBackendIdentity(t *testing.T) { &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ "exec": { - ID: "exec", - BackendID: "custom", - Model: "model", + ID: "exec", ResolvedTarget: &domain.ExecutionTarget{BackendID: "custom", + Model: "model", TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds}, }, }}, - fakeBackendResolver{backends: map[string]domain.Backend{ - "custom": {ID: "custom", Endpoint: "http://backend.example/v1"}, - }}, + defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, @@ -1505,7 +1442,7 @@ func TestRunnerAdmissionUsesResolvedBackendIdentity(t *testing.T) { &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ "exec": defaultExecutionProfile(), }}, - nil, + defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, @@ -1565,11 +1502,9 @@ func TestRunnerAdmissionFailureSkipsCompletionCollaborators(t *testing.T) { runner := NewRunnerWithRepairer( &fakePromptRepo{def: def}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": {ID: "exec", BackendID: "custom", Model: "model"}, - }}, - fakeBackendResolver{backends: map[string]domain.Backend{ - "custom": {ID: "custom", Endpoint: "http://backend.example/v1"}, + "exec": {ID: "exec", ResolvedTarget: &domain.ExecutionTarget{BackendID: "custom", Endpoint: "http://backend.example/v1", Model: "model"}}, }}, + reader, renderer, llmClient, @@ -1681,7 +1616,7 @@ func TestRunnerReleasesAdmissionAcrossRunOutcomes(t *testing.T) { &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ "exec": defaultExecutionProfile(), }}, - nil, + reader, defaultRenderer(), llmClient, @@ -1715,8 +1650,8 @@ func TestRunnerRunExplicitProfileIDIsUsed(t *testing.T) { promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} promptRepo.def.DefaultProfile = "default-prof" execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "explicit-prof": {ID: "explicit-prof", Endpoint: "http://explicit/v1", Model: "explicit"}, - "default-prof": {ID: "default-prof", Endpoint: "http://default/v1", Model: "default"}, + "explicit-prof": {ID: "explicit-prof", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://explicit/v1", Model: "explicit", TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds}}, + "default-prof": {ID: "default-prof", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://default/v1", Model: "default", TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds}}, }} runner := newMinimalRunner(promptRepo, execRepo) @@ -1740,7 +1675,7 @@ func TestRunnerRunPromptDefaultProfileIsUsedWhenNoExplicitProfileID(t *testing.T promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} promptRepo.def.DefaultProfile = "from-prompt" execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "from-prompt": {ID: "from-prompt", Endpoint: "http://llm/v1", Model: "m"}, + "from-prompt": {ID: "from-prompt", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://llm/v1", Model: "m", TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds}}, }} runner := newMinimalRunner(promptRepo, execRepo) @@ -1792,18 +1727,17 @@ func TestRunnerRunExplicitRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ "exec": { - ID: "exec", - Endpoint: "http://profile/v1", - Model: "profile-model", - Temperature: 0.2, - MaxTokens: 500, - TopP: 0.9, - TimeoutSeconds: 120, - ServiceTier: "priority", + ID: "exec", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://profile/v1", + Model: "profile-model", + Temperature: 0.2, + MaxTokens: 500, + TopP: 0.9, + TimeoutSeconds: 120, + ServiceTier: "priority"}, }, }} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}} - runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil) + runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil) res, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", @@ -1838,16 +1772,15 @@ func TestRunnerRunSelectedProfileBeatsBuiltInDefault(t *testing.T) { promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ "exec": { - ID: "exec", - Endpoint: "http://profile/v1", - Model: "profile-model", - TopP: 0.8, - TimeoutSeconds: 90, - ServiceTier: "priority", + ID: "exec", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://profile/v1", + Model: "profile-model", + TopP: 0.8, + TimeoutSeconds: 90, + ServiceTier: "priority"}, }, }} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}} - runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil) + runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil) res, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", @@ -1872,9 +1805,9 @@ func TestRunnerRunAPIKeyEnvResolvesFromEnvironment(t *testing.T) { t.Setenv("PROMPTKIT_TEST_API_KEY", "secret") promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_TEST_API_KEY"}, + "exec": {ID: "exec", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_TEST_API_KEY", TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds}}, }} - runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil, nil) + runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil, nil) res, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()}) if err != nil { @@ -1891,10 +1824,10 @@ func TestRunnerRunOptionalAPIKeyEnvMissingEnvironmentValueReachesLLM(t *testing. promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: environmentName}, + "exec": {ID: "exec", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: environmentName, TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds}}, }} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}} - runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil) + runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil) result, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()}) if err != nil || result == nil { @@ -1912,10 +1845,10 @@ func TestRunnerRunDirectAPIKeyBypassesMissingEnvAndReachesLLM(t *testing.T) { const directKey = "direct-runner-key" promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_MISSING_KEY"}, + "exec": {ID: "exec", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_MISSING_KEY", TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds}}, }} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}} - runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil) + runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil) result, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", @@ -1940,9 +1873,9 @@ func TestRunnerRunDirectAPIKeyBypassesMissingEnvAndReachesLLM(t *testing.T) { func TestRunnerPrepareAPIKeyRequiredFailsWithoutDirectKey(t *testing.T) { promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true}, + "exec": {ID: "exec", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true, TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds}}, }} - runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil, nil) + runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil, nil) _, err := runner.Prepare(context.Background(), domain.RunRequest{ PromptID: "p", @@ -1958,10 +1891,10 @@ func TestRunnerRunAPIKeyRequiredSucceedsWithDirectKey(t *testing.T) { const directKey = "direct-required-key" promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true}, + "exec": {ID: "exec", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true, TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds}}, }} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}} - runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil) + runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil, nil) _, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", @@ -1986,9 +1919,9 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideWorks(t *testing.T) { promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"}, + "exec": {ID: "exec", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://profile/v1", Model: "profile-model", TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds}}, }} - runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil, nil) + runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil, nil) res, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", @@ -2011,9 +1944,9 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideBeatsProfile(t *testing.T) { promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: profileEnv}, + "exec": {ID: "exec", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: profileEnv, TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds}}, }} - runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil, nil) + runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil, nil) res, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", @@ -2035,9 +1968,9 @@ func TestRunnerRunAPIKeyValueNotPresentInMetadata(t *testing.T) { t.Setenv(envName, secret) promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName}, + "exec": {ID: "exec", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName, TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds}}, }} - runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil, nil) + runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil, nil) res, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()}) if err != nil { @@ -2053,7 +1986,7 @@ func TestRunnerRunAPIKeyValueNotPresentInMetadata(t *testing.T) { } func TestRunnerRunPromptLoadFailure(t *testing.T) { - runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil, nil) + runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil, nil) _, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"}) if !errors.Is(err, ErrPromptLoad) { t.Fatalf("expected ErrPromptLoad, got %v", err) @@ -2066,7 +1999,7 @@ func TestRunnerRunPromptLoadFailure(t *testing.T) { func TestRunnerRunArtifactLoadFailure(t *testing.T) { runner := NewRunner( &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, &fakeArtifactReader{errByURI: map[string]error{"a://bad": errors.New("read failed")}}, &fakeRenderer{rendered: &domain.RenderedPrompt{}}, @@ -2086,7 +2019,7 @@ func TestRunnerRunArtifactLoadFailure(t *testing.T) { func TestRunnerRunPromptRenderFailure(t *testing.T) { runner := NewRunner( &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), &fakeRenderer{err: errors.New("render failed")}, @@ -2106,7 +2039,7 @@ func TestRunnerRunPromptRenderFailure(t *testing.T) { func TestRunnerRunLLMFailure(t *testing.T) { runner := NewRunner( &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), @@ -2126,7 +2059,7 @@ func TestRunnerRunLLMFailure(t *testing.T) { func TestRunnerRunCancellationPreservesGenerationCategory(t *testing.T) { runner := NewRunner( &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), @@ -2152,7 +2085,7 @@ func TestRunnerRunCancellationPreservesGenerationCategory(t *testing.T) { func TestRunnerRunLLMInvalidRequestMapsToUsecaseInvalidRequest(t *testing.T) { runner := NewRunner( &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), @@ -2176,7 +2109,7 @@ func TestRunnerRunValidationStillWorks(t *testing.T) { validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}} runner := NewRunner( &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), @@ -2343,11 +2276,9 @@ func TestRunnerRepairStateMachine(t *testing.T) { runner := NewRunnerWithRepairer( &fakePromptRepo{def: definition}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": {ID: "exec", BackendID: "custom", Model: "profile-model"}, - }}, - fakeBackendResolver{backends: map[string]domain.Backend{ - "custom": {ID: "custom", Endpoint: "http://backend.example/v1"}, + "exec": {ID: "exec", ResolvedTarget: &domain.ExecutionTarget{BackendID: "custom", Endpoint: "http://backend.example/v1", Model: "profile-model"}}, }}, + defaultArtifactReader(), defaultRenderer(), client, @@ -2483,11 +2414,9 @@ func TestRunnerSchedulesInitialAndRepairGenerationThroughOneBackendPool(t *testi runner := NewRunnerWithRepairer( staticPromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)}, staticExecutionProfileRepo{profile: &domain.ExecutionProfile{ - ID: "exec", BackendID: "custom", Model: "model", - }}, - fakeBackendResolver{backends: map[string]domain.Backend{ - "custom": {ID: "custom", Endpoint: "http://backend.example/v1"}, + ID: "exec", ResolvedTarget: &domain.ExecutionTarget{BackendID: "custom", Endpoint: "http://backend.example/v1", Model: "model"}, }}, + staticArtifactReader{}, staticRenderer{}, scheduledClient, @@ -2556,220 +2485,6 @@ func TestRunnerSchedulesInitialAndRepairGenerationThroughOneBackendPool(t *testi } } -func TestExecutionProfileToTargetPopulatesAllFieldsAndCopiesExtraParams(t *testing.T) { - src := &domain.ExecutionProfile{ - ID: "exec", - Endpoint: "http://profile/v1", - Model: "profile-model", - Temperature: 0.2, - MaxTokens: 123, - TopP: 0.75, - TimeoutSeconds: 90, - ServiceTier: "priority", - ReasoningEffort: "medium", - APIKeyEnv: "PROMPTKIT_API_KEY", - APIKeyRequired: true, - ExtraParams: map[string]any{ - "provider_option": "on", - }, - } - - target := executionProfileToTarget(src) - if target.Endpoint != src.Endpoint || - target.Model != src.Model || - target.Temperature != src.Temperature || - target.MaxTokens != src.MaxTokens || - target.TopP != src.TopP || - target.TimeoutSeconds != src.TimeoutSeconds || - target.ServiceTier != src.ServiceTier || - target.ReasoningEffort != src.ReasoningEffort || - target.APIKeyEnv != src.APIKeyEnv || - target.APIKeyRequired != src.APIKeyRequired { - t.Fatalf("expected all profile fields to populate target, got %+v", target) - } - if !reflect.DeepEqual(target.ExtraParams, src.ExtraParams) { - t.Fatalf("expected extra_params to match, got %#v", target.ExtraParams) - } - - src.ExtraParams["provider_option"] = "changed" - if target.ExtraParams["provider_option"] != "on" { - t.Fatalf("expected extra_params copy to be independent, got %#v", target.ExtraParams) - } -} - -func TestResolveExecutionTargetProfileValuesPopulateAllSupportedFields(t *testing.T) { - profileValue := &domain.ExecutionProfile{ - ID: "exec", - Endpoint: "http://profile/v1", - Model: "profile-model", - Temperature: 0.3, - MaxTokens: 222, - TopP: 0.6, - TimeoutSeconds: 77, - ServiceTier: "priority", - ReasoningEffort: "low", - APIKeyEnv: "PROFILE_KEY", - ExtraParams: map[string]any{ - "profile_option": "enabled", - }, - } - - target, presence := resolveExecutionTarget(nil, profileValue, nil) - if presence != (domain.ExecutionTargetPresence{}) { - t.Fatalf("expected no request override presence, got %+v", presence) - } - if target.Endpoint != profileValue.Endpoint || - target.Model != profileValue.Model || - target.Temperature != profileValue.Temperature || - target.MaxTokens != profileValue.MaxTokens || - target.TopP != profileValue.TopP || - target.TimeoutSeconds != profileValue.TimeoutSeconds || - target.ServiceTier != profileValue.ServiceTier || - target.ReasoningEffort != profileValue.ReasoningEffort || - target.APIKeyEnv != profileValue.APIKeyEnv || - target.APIKeyRequired != profileValue.APIKeyRequired { - t.Fatalf("expected profile values to populate target, got %+v", target) - } - if !reflect.DeepEqual(target.ExtraParams, profileValue.ExtraParams) { - t.Fatalf("expected profile extra_params in target, got %#v", target.ExtraParams) - } -} - -func TestResolveExecutionTargetRuntimeOverridesBeatProfileForAllOverrideableFields(t *testing.T) { - profileValue := &domain.ExecutionProfile{ - ID: "exec", - Endpoint: "http://profile/v1", - Model: "profile-model", - Temperature: 0.2, - MaxTokens: 200, - TopP: 0.8, - TimeoutSeconds: 90, - ServiceTier: "priority", - ReasoningEffort: "medium", - APIKeyEnv: "PROFILE_KEY", - ExtraParams: map[string]any{ - "profile_only": "yes", - }, - } - override := &domain.ExecutionTargetOverride{ - Endpoint: "http://override/v1", - Model: "override-model", - Temperature: float64Ptr(0.9), - MaxTokens: intPtr(111), - TopP: float64Ptr(0.5), - TimeoutSeconds: intPtr(30), - ServiceTier: "flex", - ReasoningEffort: stringPtr("high"), - APIKeyEnv: "RUNTIME_KEY", - ExtraParams: map[string]any{ - "runtime_only": "yes", - }, - } - - target, presence := resolveExecutionTarget(nil, profileValue, override) - if presence != (domain.ExecutionTargetPresence{Temperature: true, MaxTokens: true, TopP: true, TimeoutSeconds: true}) { - t.Fatalf("unexpected override presence: %+v", presence) - } - if target.Endpoint != override.Endpoint || - target.Model != override.Model || - target.Temperature != *override.Temperature || - target.MaxTokens != *override.MaxTokens || - target.TopP != *override.TopP || - target.TimeoutSeconds != *override.TimeoutSeconds || - target.ServiceTier != override.ServiceTier || - target.ReasoningEffort != *override.ReasoningEffort || - target.APIKeyEnv != override.APIKeyEnv { - t.Fatalf("expected runtime overrides to win for all fields, got %+v", target) - } - if !reflect.DeepEqual(target.ExtraParams, override.ExtraParams) { - t.Fatalf("expected runtime extra_params to replace profile extra_params, got %#v", target.ExtraParams) - } -} - -func TestResolveExecutionTargetReasoningOverrideStates(t *testing.T) { - profileValue := &domain.ExecutionProfile{ - ReasoningEffort: "medium", - } - tests := []struct { - name string - override *string - want string - }{ - { - name: "nil inherits profile value", - want: "medium", - }, - { - name: "nonblank replaces and trims profile value", - override: stringPtr(" high "), - want: "high", - }, - { - name: "blank clears profile value", - override: stringPtr(" \t "), - want: "", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - target, _ := resolveExecutionTarget(nil, profileValue, &domain.ExecutionTargetOverride{ - ReasoningEffort: tt.override, - }) - if target.ReasoningEffort != tt.want { - t.Fatalf("reasoning effort = %q, want %q", target.ReasoningEffort, tt.want) - } - }) - } -} - -func TestMergeExecutionTargetEmptyStringOverridesDoNotErase(t *testing.T) { - base := domain.ExecutionTarget{ - Endpoint: "http://base/v1", - Model: "base-model", - ServiceTier: "priority", - ReasoningEffort: "medium", - APIKeyEnv: "BASE_KEY", - } - override := domain.ExecutionTarget{ - Endpoint: "http://override/v1", - Model: "override-model", - ServiceTier: " ", - ReasoningEffort: " ", - APIKeyEnv: "", - } - - merged := mergeExecutionTarget(base, override) - if merged.Endpoint != "http://override/v1" || merged.Model != "override-model" { - t.Fatalf("expected endpoint/model to override, got %+v", merged) - } - if merged.ServiceTier != "priority" { - t.Fatalf("expected empty service_tier override to be ignored, got %q", merged.ServiceTier) - } - if merged.ReasoningEffort != "medium" { - t.Fatalf("expected empty reasoning_effort override to be ignored, got %q", merged.ReasoningEffort) - } - if merged.APIKeyEnv != "BASE_KEY" { - t.Fatalf("expected empty api_key_env override to be ignored, got %q", merged.APIKeyEnv) - } -} - -func TestMergeExecutionTargetEmptyExtraParamsDoesNotErase(t *testing.T) { - base := domain.ExecutionTarget{ - ExtraParams: map[string]any{ - "keep": "value", - }, - } - override := domain.ExecutionTarget{ - ExtraParams: map[string]any{}, - } - - merged := mergeExecutionTarget(base, override) - if !reflect.DeepEqual(merged.ExtraParams, base.ExtraParams) { - t.Fatalf("expected empty extra_params override not to erase base values, got %#v", merged.ExtraParams) - } -} - func TestBuildOutputArtifactDefaults(t *testing.T) { tests := []struct { name string @@ -2817,87 +2532,19 @@ func hashString(s string) string { func defaultExecutionProfile() *domain.ExecutionProfile { return &domain.ExecutionProfile{ - ID: "exec", - Endpoint: "http://llm/v1", - Model: "model-from-profile", + ID: "exec", ResolvedTarget: &domain.ExecutionTarget{Endpoint: "http://llm/v1", + Model: "model-from-profile", TimeoutSeconds: defaults.ExecutionDefaultTimeoutSeconds}, } } -func TestResolveExecutionTargetUsesBackendProfileAndRequestPrecedence(t *testing.T) { - backendValue := &domain.Backend{ - ID: "custom", - Endpoint: "http://backend/v1", - APIKeyEnv: "BACKEND_KEY", - ExtraParams: map[string]any{"backend": true}, - } - profileValue := &domain.ExecutionProfile{ - ID: "exec", - BackendID: "custom", - Endpoint: "http://profile/v1", - Model: "profile-model", - APIKeyEnv: "PROFILE_KEY", - ExtraParams: map[string]any{"profile": true}, - } - override := &domain.ExecutionTargetOverride{ - Endpoint: "http://request/v1", - APIKeyEnv: "REQUEST_KEY", - ExtraParams: map[string]any{"request": true}, - } - - target, _ := resolveExecutionTarget(backendValue, profileValue, override) - if target.BackendID != "custom" { - t.Fatalf("endpoint override changed backend identity: %+v", target) - } - if target.Endpoint != "http://request/v1" || target.APIKeyEnv != "REQUEST_KEY" { - t.Fatalf("request values did not win: %+v", target) - } - if !reflect.DeepEqual(target.ExtraParams, map[string]any{"request": true}) { - t.Fatalf("expected whole-map request replacement, got %#v", target.ExtraParams) - } - - target, _ = resolveExecutionTarget(backendValue, &domain.ExecutionProfile{ - ID: "exec", BackendID: "custom", Model: "profile-model", - }, nil) - if target.Endpoint != backendValue.Endpoint || - target.APIKeyEnv != backendValue.APIKeyEnv || - !reflect.DeepEqual(target.ExtraParams, backendValue.ExtraParams) { - t.Fatalf("backend defaults were not inherited: %+v", target) - } -} - -func TestRunnerPrepareBackendResolutionAndCredentialPrecedence(t *testing.T) { - resolver := fakeBackendResolver{backends: map[string]domain.Backend{ - "custom": {ID: "custom", Endpoint: "http://backend/v1", APIKeyEnv: "BACKEND_KEY"}, - }} +func TestRunnerPrepareCredentialPrecedence(t *testing.T) { promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} - t.Run("unknown backend is a profile load failure", func(t *testing.T) { - runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": {ID: "exec", BackendID: "unknown", Model: "model"}, - }}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil, nil) - - _, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()}) - if !errors.Is(err, ErrProfileLoad) { - t.Fatalf("expected ErrProfileLoad, got %v", err) - } - }) - - t.Run("nil resolver is a profile load failure", func(t *testing.T) { - runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": {ID: "exec", BackendID: "custom", Model: "model"}, - }}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil, nil) - - _, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()}) - if !errors.Is(err, ErrProfileLoad) { - t.Fatalf("expected ErrProfileLoad, got %v", err) - } - }) - t.Run("request environment wins", func(t *testing.T) { t.Setenv("REQUEST_KEY", "request-secret") runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": {ID: "exec", BackendID: "custom", Model: "model", APIKeyEnv: "PROFILE_KEY"}, - }}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil, nil) + "exec": {ID: "exec", ResolvedTarget: &domain.ExecutionTarget{BackendID: "custom", Endpoint: "http://backend/v1", Model: "model", APIKeyEnv: "PROFILE_KEY"}}, + }}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil, nil) prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(), @@ -2914,8 +2561,8 @@ func TestRunnerPrepareBackendResolutionAndCredentialPrecedence(t *testing.T) { t.Run("required direct key clears inherited environment", func(t *testing.T) { t.Setenv("BACKEND_KEY", "backend-secret") runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": {ID: "exec", BackendID: "custom", Model: "model", APIKeyRequired: true}, - }}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil, nil) + "exec": {ID: "exec", ResolvedTarget: &domain.ExecutionTarget{BackendID: "custom", Endpoint: "http://backend/v1", Model: "model", APIKeyRequired: true}}, + }}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil, nil) _, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()}) if !errors.Is(err, ErrAPIKeyRequired) { @@ -2962,7 +2609,7 @@ func intPtr(v int) *int { func newMinimalRunner(promptRepo *fakePromptRepo, execRepo *fakeExecutionProfileRepo) *Runner { return NewRunner( promptRepo, - execRepo, nil, + execRepo, defaultArtifactReader(), defaultRenderer(), diff --git a/observation_boundary_contract_test.go b/observation_boundary_contract_test.go index 6f552d8..2a09984 100644 --- a/observation_boundary_contract_test.go +++ b/observation_boundary_contract_test.go @@ -7,8 +7,8 @@ import ( "testing" "time" + "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/promptkit" - "gitea.maximumdirect.net/eric/promptkit/testfixture" ) type observationClientFunc func(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) @@ -24,16 +24,22 @@ func TestDelegatedOperationsIsolateObservations(t *testing.T) { ctx, cancel := context.WithTimeout(context.WithValue(context.Background(), contextKey{}, "caller-value"), time.Minute) defer cancel() deadline, _ := ctx.Deadline() - child, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("child", "profile", "child message"), "."), - testfixture.WithProfiles(testfixture.Profile{ID: "profile", Endpoint: "https://example.invalid/v1", Model: "child-model"}), - promptkit.WithLLMClient(observationClientFunc(func(callCtx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { - gotDeadline, ok := callCtx.Deadline() - if !ok || gotDeadline != deadline || callCtx.Value(contextKey{}) != "caller-value" || callCtx.Done() != ctx.Done() { - t.Error("delegation lost caller context behavior") - } - return &promptkit.GenerateResponse{Content: "child", Usage: promptkit.TokenUsage{PromptTokens: 7}}, nil - }))) + child, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{ + Source: configkit.Mapping("child execution", map[string]any{ + "profiles": map[string]any{"profile": map[string]any{"endpoint": "https://example.invalid/v1", "model": "child-model"}}, + "prompts": map[string]any{"child": map[string]any{"1": map[string]any{ + "default_profile": "profile", + "messages": []any{map[string]any{"role": "user", "content": "child message"}}, + "output": map[string]any{"format": "text", "validation_mode": "none"}, + }}}, + }), + }}}, promptkit.WithLLMClient(observationClientFunc(func(callCtx context.Context, _ promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + gotDeadline, ok := callCtx.Deadline() + if !ok || gotDeadline != deadline || callCtx.Value(contextKey{}) != "caller-value" || callCtx.Done() != ctx.Done() { + t.Error("delegation lost caller context behavior") + } + return &promptkit.GenerateResponse{Content: "child", Usage: promptkit.TokenUsage{PromptTokens: 7}}, nil + }))) if err != nil { t.Fatal(err) } diff --git a/observation_contract_test.go b/observation_contract_test.go index abe9fac..45c9019 100644 --- a/observation_contract_test.go +++ b/observation_contract_test.go @@ -6,22 +6,13 @@ import ( "errors" "strings" "testing" - "testing/fstest" "gitea.maximumdirect.net/eric/promptkit" - "gitea.maximumdirect.net/eric/promptkit/testfixture" ) func observationEngine(t *testing.T, client promptkit.LLMClient) *promptkit.Engine { t.Helper() - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "hello"), "."), - testfixture.WithProfiles(testfixture.Profile{ID: "profile", Endpoint: "https://example.invalid/v1", Model: "model"}), - promptkit.WithLLMClient(client)) - if err != nil { - t.Fatal(err) - } - return engine + return newSimpleEngine(t, "hello", promptkit.WithLLMClient(client)) } func TestRunObservationCoversFacadeErrors(t *testing.T) { @@ -129,7 +120,7 @@ func TestRunObservationClassifiesInvalidOutputAndGenerationFailure(t *testing.T) } func TestRunObservationPromptFailureAndPanicCleanup(t *testing.T) { - engine, err := testfixture.NewEngine(testfixture.Config{}, testfixture.WithPromptFS(fstest.MapFS{}, ".")) + engine, err := promptkit.NewEngine(promptkit.Config{}) if err != nil { t.Fatal(err) } diff --git a/output_contract_contract_test.go b/output_contract_contract_test.go index d7cd7ad..0c82b5e 100644 --- a/output_contract_contract_test.go +++ b/output_contract_contract_test.go @@ -6,22 +6,10 @@ import ( "testing" "gitea.maximumdirect.net/eric/promptkit" - "gitea.maximumdirect.net/eric/promptkit/testfixture" ) func TestPreparationRejectsInvalidOutputContractWithPublicError(t *testing.T) { - engine, err := testfixture.NewEngine( - testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), - testfixture.WithProfiles(testfixture.Profile{ - ID: "profile", - Endpoint: "http://example.test/v1", - Model: "model", - }), - ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } + engine := newSimpleEngine(t, "message") for _, tc := range []struct { name string diff --git a/prepared_execution_contract_test.go b/prepared_execution_contract_test.go index 357170f..cc2e89d 100644 --- a/prepared_execution_contract_test.go +++ b/prepared_execution_contract_test.go @@ -12,8 +12,8 @@ import ( "testing/fstest" "time" + "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/promptkit" - "gitea.maximumdirect.net/eric/promptkit/testfixture" ) func TestPreparedExecutionFreezesSourcesAndReturnsIndependentDetails(t *testing.T) { @@ -27,10 +27,17 @@ func TestPreparedExecutionFreezesSourcesAndReturnsIndependentDetails(t *testing. client := &preparedRecordingClient{ response: &promptkit.GenerateResponse{Content: `{"value":3}`}, } - engine, err := testfixture.NewEngine( - testfixture.Config{}, - testfixture.WithPromptFS(promptSource, "."), - testfixture.WithProfileFS(profileSource, "."), + promptInputs, err := promptkit.FilesystemInputs("prompts", promptSource, ".") + if err != nil { + t.Fatal(err) + } + profileInputs, err := promptkit.FilesystemInputs("profiles", profileSource, ".") + if err != nil { + t.Fatal(err) + } + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{ + {Inputs: profileInputs}, {Inputs: promptInputs}, + }}}, promptkit.WithSchemaFS(schemaSource, "."), promptkit.WithArtifactReader(reader), promptkit.WithLLMClient(client), @@ -431,17 +438,7 @@ func TestPreparedExecutionDiscardAndFormattingDoNotExposePrivateState(t *testing client := &preparedRecordingClient{ response: &promptkit.GenerateResponse{Content: "generated output"}, } - engine, err := testfixture.NewEngine( - testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prepared", "profile", renderedContent), "."), - testfixture.WithProfiles(testfixture.Profile{ - ID: "profile", - Endpoint: "http://example.test/v1", - Model: "model", - APIKeyRequired: true, - }), - promptkit.WithLLMClient(client), - ) + engine, err := promptkit.NewEngine(preparedContractConfig(renderedContent, true), promptkit.WithLLMClient(client)) if err != nil { t.Fatalf("construct engine: %v", err) } @@ -565,17 +562,7 @@ func TestPreparedExecutionCredentialCapacityAndTimingBoundaries(t *testing.T) { client := &preparedRecordingClient{ response: &promptkit.GenerateResponse{Content: "unexpected"}, } - engine, err := testfixture.NewEngine( - testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prepared", "profile", "content"), "."), - testfixture.WithProfiles(testfixture.Profile{ - ID: "profile", - Endpoint: "http://example.test/v1", - Model: "model", - APIKeyRequired: true, - }), - promptkit.WithLLMClient(client), - ) + engine, err := promptkit.NewEngine(preparedContractConfig("content", true), promptkit.WithLLMClient(client)) if err != nil { t.Fatalf("construct credential engine: %v", err) } @@ -767,47 +754,55 @@ func newPreparedContractEngine( message string, ) *promptkit.Engine { t.Helper() - engine, err := testfixture.NewEngine( - testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prepared", "profile", message), "."), - testfixture.WithProfiles(testfixture.Profile{ - ID: "profile", - Endpoint: "http://example.test/v1", - Model: "model", - }), - promptkit.WithLLMClient(client), - ) + engine, err := promptkit.NewEngine(preparedContractConfig(message, false), promptkit.WithLLMClient(client)) if err != nil { t.Fatalf("construct prepared execution engine: %v", err) } return engine } +func preparedContractConfig(message string, keyRequired bool) promptkit.Config { + return promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{ + Source: configkit.Mapping("prepared execution", map[string]any{ + "profiles": map[string]any{"profile": map[string]any{ + "endpoint": "http://example.test/v1", "model": "model", "api_key_required": keyRequired, + }}, + "prompts": map[string]any{"prepared": map[string]any{"1": map[string]any{ + "default_profile": "profile", + "messages": []any{map[string]any{"role": "user", "content": message}}, + "output": map[string]any{"format": "text", "validation_mode": "none"}, + }}}, + }), + }}} +} + func preparedPromptSource(label string) fstest.MapFS { return fstest.MapFS{ - "prompt.yaml": &fstest.MapFile{Data: []byte(`id: prepared -version: "1" -default_profile: profile -inputs: - - name: input - required: true -messages: - - role: user - content: 'Input={{input "input"}} Label={{.label}}' -description: ` + label + ` -output: - format: json - validation_mode: json_schema - schema_path: schema.json + "prompt.yaml": &fstest.MapFile{Data: []byte(`prompts: + prepared: + "1": + default_profile: profile + inputs: + - name: input + required: true + messages: + - role: user + content: 'Input={{input "input"}} Label={{.label}}' + description: ` + label + ` + output: + format: json + validation_mode: json_schema + schema_path: schema.json `)}, } } func preparedProfileSource(model string) fstest.MapFS { return fstest.MapFS{ - "profile.yaml": &fstest.MapFile{Data: []byte(`id: profile -endpoint: http://example.test/v1 -model: ` + model + ` + "profile.yaml": &fstest.MapFile{Data: []byte(`profiles: + profile: + endpoint: http://example.test/v1 + model: ` + model + ` `)}, } } diff --git a/prepared_observation_contract_test.go b/prepared_observation_contract_test.go index 2c8033c..5ef59db 100644 --- a/prepared_observation_contract_test.go +++ b/prepared_observation_contract_test.go @@ -6,8 +6,8 @@ import ( "sync" "testing" + "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/promptkit" - "gitea.maximumdirect.net/eric/promptkit/testfixture" ) func TestPreparedObservationStartsOnlyAfterClaimAndKeepsObserver(t *testing.T) { @@ -132,10 +132,18 @@ func TestPreparedObservationClaimRaceAndPanicCleanup(t *testing.T) { func TestPreparedObservationFinishesAfterCredentialAndCapacityFailures(t *testing.T) { const envName = "PROMPTKIT_PREPARED_OBSERVATION_KEY" t.Setenv(envName, "available") - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prepared", "profile", "hello"), "."), - testfixture.WithProfiles(testfixture.Profile{ID: "profile", Endpoint: "https://example.invalid/v1", Model: "model", APIKeyRequired: true}), - promptkit.WithLLMClient(&fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}})) + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{ + Source: configkit.Mapping("credential preparation", map[string]any{ + "profiles": map[string]any{"profile": map[string]any{ + "endpoint": "https://example.invalid/v1", "model": "model", "api_key_required": true, + }}, + "prompts": map[string]any{"prepared": map[string]any{"1": map[string]any{ + "default_profile": "profile", + "messages": []any{map[string]any{"role": "user", "content": "hello"}}, + "output": map[string]any{"format": "text", "validation_mode": "none"}, + }}}, + }), + }}}, promptkit.WithLLMClient(&fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}})) if err != nil { t.Fatal(err) } diff --git a/profile_inheritance_contract_test.go b/profile_inheritance_contract_test.go index 3b8cc12..a2cdafb 100644 --- a/profile_inheritance_contract_test.go +++ b/profile_inheritance_contract_test.go @@ -4,247 +4,169 @@ import ( "context" "errors" "io/fs" - "strings" "sync" "testing" "testing/fstest" + "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/promptkit" - "gitea.maximumdirect.net/eric/promptkit/testfixture" ) +func aliasPrompt() map[string]any { + return map[string]any{"prompts": map[string]any{"prepared": map[string]any{"1": map[string]any{ + "default_profile": "child", + "messages": []any{map[string]any{"role": "user", "content": "hello"}}, + "output": map[string]any{"format": "text", "validation_mode": "none"}, + }}}} +} + func TestProfileInheritanceBuiltInAliasWorkflow(t *testing.T) { - engine, err := testfixture.NewEngine(testfixture.Config{ - PromptDir: frameworkPromptDir, - SchemaDir: frameworkSchemaDir, - }, testfixture.WithProfiles(testfixture.Profile{ - ID: "weather-light", - BaseProfileID: "promptkit-openrouter-standard", - ReasoningEffort: "high", - TimeoutSeconds: 120, - })) + standardID, reasoning, timeout := "promptkit-openrouter-standard", "high", 120 + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ + Underlays: []configkit.Underlay{{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("application alias", map[string]promptkit.ProfileConfig{ + "child": {Extends: &standardID, ReasoningEffort: &reasoning, TimeoutSeconds: &timeout}, + })}}}}, + Main: &configkit.Input{Source: configkit.Mapping("prompt", aliasPrompt())}, + }}) if err != nil { t.Fatalf("construct alias engine: %v", err) } - - base, err := engine.InspectProfile(context.Background(), "promptkit-openrouter-standard") + base, err := engine.InspectProfile(context.Background(), standardID) if err != nil { t.Fatalf("inspect base: %v", err) } - child, err := engine.InspectProfile(context.Background(), "weather-light") + child, err := engine.InspectProfile(context.Background(), "child") if err != nil { t.Fatalf("inspect alias: %v", err) } - if child.ProfileID != "weather-light" || - child.EffectiveModelParams.BackendID != base.EffectiveModelParams.BackendID || + if child.ProfileID != "child" || child.EffectiveModelParams.BackendID != base.EffectiveModelParams.BackendID || child.EffectiveModelParams.Model != base.EffectiveModelParams.Model || - child.EffectiveModelParams.ReasoningEffort != "high" || - child.EffectiveModelParams.TimeoutSeconds != 120 { + child.EffectiveModelParams.ReasoningEffort != reasoning || child.EffectiveModelParams.TimeoutSeconds != timeout { t.Fatalf("alias inspection = %+v, base = %+v", child, base) } - - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "weather-light", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - }) - if err != nil { - t.Fatalf("prepare alias: %v", err) + prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prepared"}) + if err != nil || prepared.SelectedProfileID != "child" { + t.Fatalf("prepare alias = (%+v, %v)", prepared, err) } - if prepared.SelectedProfileID != "weather-light" { - t.Fatalf("SelectedProfileID = %q", prepared.SelectedProfileID) - } - - timeout := 15 - reasoning := "low" - overridden, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - ProfileID: "weather-light", - Execution: &promptkit.ExecutionTargetOverride{ - TimeoutSeconds: &timeout, - ReasoningEffort: &reasoning, - }, - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - }) - if err != nil { - t.Fatalf("prepare override: %v", err) - } - if overridden.EffectiveModelParams.TimeoutSeconds != timeout || - overridden.EffectiveModelParams.ReasoningEffort != reasoning { - t.Fatalf("runtime override target = %+v", overridden.EffectiveModelParams) + shorter, lower := 15, "low" + overridden, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prepared", Execution: &promptkit.ExecutionTargetOverride{ + TimeoutSeconds: &shorter, ReasoningEffort: &lower, + }}) + if err != nil || overridden.EffectiveModelParams.TimeoutSeconds != shorter || overridden.EffectiveModelParams.ReasoningEffort != lower { + t.Fatalf("runtime override = (%+v, %v)", overridden, err) } } func TestProfileInheritanceYAMLAliasOfBuiltIn(t *testing.T) { - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(fstest.MapFS{}, "."), - testfixture.WithProfileFS(fstest.MapFS{ - "alias.yaml": &fstest.MapFile{Data: []byte("id: yaml-alias\nbase_profile: promptkit-openrouter-standard\n")}, - }, "."), - ) + inputs, err := promptkit.FilesystemInputs("profile alias", fstest.MapFS{ + "config/alias.yaml": &fstest.MapFile{Data: []byte("profiles:\n yaml-alias:\n extends: promptkit-openrouter-standard\n")}, + }, "config") if err != nil { - t.Fatalf("construct YAML alias engine: %v", err) + t.Fatal(err) + } + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{{Inputs: inputs}}}}) + if err != nil { + t.Fatal(err) } inspection, err := engine.InspectProfile(context.Background(), "yaml-alias") - if err != nil { - t.Fatalf("inspect YAML alias: %v", err) - } - if inspection.ProfileID != "yaml-alias" || inspection.EffectiveModelParams.Model == "" { - t.Fatalf("YAML alias inspection = %+v", inspection) + if err != nil || inspection.ProfileID != "yaml-alias" || inspection.EffectiveModelParams.Model == "" { + t.Fatalf("YAML alias inspection = (%+v, %v)", inspection, err) } } func TestProfileInheritanceRetainsRequiredCredentialBehavior(t *testing.T) { - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(fstest.MapFS{}, "."), - testfixture.WithBackend(testfixture.Backend{ - ID: "credential-backend", - Endpoint: "https://credential.example/v1", - APIKeyEnv: "OPTIONAL_BACKEND_KEY", - }), - testfixture.WithProfiles( - testfixture.Profile{ - ID: "credential-base", - BackendID: "credential-backend", - Model: "model", - APIKeyRequired: true, + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{ + Source: configkit.Mapping("credentials", map[string]any{ + "backends": map[string]any{"credential-backend": map[string]any{ + "endpoint": "https://credential.example/v1", "api_key_env": "OPTIONAL_BACKEND_KEY", + }}, + "profiles": map[string]any{ + "credential-base": map[string]any{"backend": "credential-backend", "model": "model", "api_key_required": true}, + "credential-child": map[string]any{"extends": "credential-base"}, }, - testfixture.Profile{ID: "credential-child", BaseProfileID: "credential-base"}, - ), - ) + }), + }}}) if err != nil { - t.Fatalf("construct credential inheritance engine: %v", err) + t.Fatal(err) } inspection, err := engine.InspectProfile(context.Background(), "credential-child") - if err != nil { - t.Fatalf("inspect credential child: %v", err) - } - if !inspection.APIKeyRequired || inspection.EffectiveModelParams.APIKeyEnv != "" { - t.Fatalf("credential inspection = %+v", inspection) + if err != nil || !inspection.APIKeyRequired || inspection.EffectiveModelParams.APIKeyEnv != "" { + t.Fatalf("credential inspection = (%+v, %v)", inspection, err) } } func TestProfileInheritancePreservesPublicErrorIdentities(t *testing.T) { - newEngine := func(t *testing.T, profiles fs.FS) (*promptkit.Engine, error) { - t.Helper() - options := []any{testfixture.WithPromptFS(fstest.MapFS{}, ".")} - if profiles != nil { - options = append(options, testfixture.WithProfileFS(profiles, ".")) + engine, err := promptkit.NewEngine(promptkit.Config{}) + if err != nil { + t.Fatal(err) + } + if inspection, err := engine.InspectProfile(context.Background(), "missing"); inspection != nil || !errors.Is(err, promptkit.ErrProfileNotFound) { + t.Fatalf("missing profile = (%+v, %v)", inspection, err) + } + for _, document := range []map[string]any{ + {"profiles": map[string]any{"child": map[string]any{"extends": "missing"}}}, + {"profiles": map[string]any{"a": map[string]any{"extends": "b"}, "b": map[string]any{"extends": "a"}}}, + } { + invalid, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{ + Source: configkit.Mapping("invalid inheritance", document), + }}}) + if invalid != nil || !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("invalid inheritance = (%+v, %v)", invalid, err) } - return testfixture.NewEngine(testfixture.Config{}, options...) - } - - tests := []struct { - name string - profiles fs.FS - profile string - contains []string - want error - wantNot error - }{ - { - name: "missing selected profile", - profile: "missing", - want: promptkit.ErrProfileNotFound, - wantNot: promptkit.ErrProfileLoad, - }, - { - name: "missing base", - profiles: fstest.MapFS{ - "child.yaml": &fstest.MapFile{Data: []byte("id: child\nbase_profile: missing\n")}, - }, - profile: "child", - contains: []string{"child", "missing"}, - want: promptkit.ErrProfileLoad, - wantNot: promptkit.ErrProfileNotFound, - }, - { - name: "cycle", - profiles: fstest.MapFS{ - "a.yaml": &fstest.MapFile{Data: []byte("id: a\nbase_profile: b\n")}, - "b.yaml": &fstest.MapFile{Data: []byte("id: b\nbase_profile: a\n")}, - }, - profile: "a", - contains: []string{"a", "b"}, - want: promptkit.ErrProfileLoad, - }, - } - - for _, tc := range tests { - t.Run(tc.name, func(t *testing.T) { - engine, err := newEngine(t, tc.profiles) - if tc.profiles != nil { - if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("construction = (%#v, %v), want invalid configuration", engine, err) - } - return - } - if err != nil { - t.Fatalf("construct engine: %v", err) - } - result, err := engine.InspectProfile(context.Background(), tc.profile) - if result != nil || !errors.Is(err, tc.want) || (tc.wantNot != nil && errors.Is(err, tc.wantNot)) { - t.Fatalf("inspection=(%+v, %v), want %v without %v", result, err, tc.want, tc.wantNot) - } - for _, fragment := range tc.contains { - if !strings.Contains(err.Error(), fragment) { - t.Fatalf("error = %v, want %q", err, fragment) - } - } - }) } } func TestProfileInheritanceFreezesPreparedExecution(t *testing.T) { profiles := &mutableInheritanceProfileFS{files: fstest.MapFS{ - "child.yaml": &fstest.MapFile{Data: []byte("id: child\nbase_profile: base\n")}, - "base.yaml": &fstest.MapFile{Data: []byte("id: base\nendpoint: https://base.example/v1\nmodel: first-model\n")}, + "config/profiles.yaml": &fstest.MapFile{Data: []byte("profiles:\n base:\n endpoint: https://base.example/v1\n model: first-model\n child:\n extends: base\n")}, }} - client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}} - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prepared", "child", "content"), "."), - testfixture.WithProfileFS(profiles, "."), - promptkit.WithLLMClient(client), - ) + inputs, err := promptkit.FilesystemInputs("profiles", profiles, "config") if err != nil { - t.Fatalf("construct engine: %v", err) + t.Fatal(err) + } + client := &recordingProfileClient{} + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ + Underlays: []configkit.Underlay{{Inputs: inputs}}, + Main: &configkit.Input{Source: configkit.Mapping("prompt", aliasPrompt())}, + }}, promptkit.WithLLMClient(client)) + if err != nil { + t.Fatal(err) } - prepared, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prepared"}) if err != nil { - t.Fatalf("prepare execution: %v", err) + t.Fatal(err) } - profiles.set("base.yaml", "id: base\nendpoint: https://base.example/v1\nmodel: second-model\n") - + profiles.set("config/profiles.yaml", "profiles:\n base:\n endpoint: https://base.example/v1\n model: second-model\n child:\n extends: base\n") result, err := engine.RunPrepared(context.Background(), prepared) - if err != nil || result == nil || len(client.requests) != 1 || client.requests[0].Target.Model != "first-model" { - t.Fatalf("prepared execution=(%+v, %v), requests=%+v", result, err, client.requests) + if err != nil || result == nil || client.model != "first-model" { + t.Fatalf("prepared execution = (%+v, %v), model %q", result, err, client.model) } inspection, err := engine.InspectProfile(context.Background(), "child") if err != nil || inspection.EffectiveModelParams.Model != "first-model" { - t.Fatalf("existing engine inspection=(%+v, %v)", inspection, err) + t.Fatalf("existing engine inspection = (%+v, %v)", inspection, err) } } +type recordingProfileClient struct{ model string } + +func (client *recordingProfileClient) Generate(ctx context.Context, request promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { + client.model = request.Target.Model + return &promptkit.GenerateResponse{Content: "ok"}, ctx.Err() +} + type mutableInheritanceProfileFS struct { mu sync.RWMutex files fstest.MapFS } -func (f *mutableInheritanceProfileFS) Open(name string) (fs.File, error) { - f.mu.RLock() - defer f.mu.RUnlock() - return f.files.Open(name) +func (source *mutableInheritanceProfileFS) Open(name string) (fs.File, error) { + source.mu.RLock() + defer source.mu.RUnlock() + return source.files.Open(name) } -func (f *mutableInheritanceProfileFS) set(name, content string) { - f.mu.Lock() - defer f.mu.Unlock() - f.files[name] = &fstest.MapFile{Data: []byte(content)} +func (source *mutableInheritanceProfileFS) set(name, content string) { + source.mu.Lock() + defer source.mu.Unlock() + source.files[name] = &fstest.MapFile{Data: []byte(content)} } diff --git a/profile_inspection_contract_test.go b/profile_inspection_contract_test.go new file mode 100644 index 0000000..6f57103 --- /dev/null +++ b/profile_inspection_contract_test.go @@ -0,0 +1,99 @@ +package promptkit_test + +import ( + "context" + "errors" + "testing" + "testing/fstest" + + "gitea.maximumdirect.net/eric/ConfigKit" + "gitea.maximumdirect.net/eric/promptkit" +) + +func TestInspectProfileUsesLiteralFileKeyForPreparation(t *testing.T) { + profiles, err := promptkit.FilesystemInputs("literal profile", fstest.MapFS{ + "config/profile.yaml": &fstest.MapFile{Data: []byte("profiles:\n ' literal ':\n endpoint: https://example.test/v1\n model: model\n")}, + }, "config") + if err != nil { + t.Fatal(err) + } + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{ + Underlays: []configkit.Underlay{{Inputs: profiles}}, + Main: &configkit.Input{Source: configkit.Mapping("prompt", map[string]any{ + "prompts": map[string]any{"prompt": map[string]any{"1": map[string]any{ + "default_profile": " literal ", + "messages": []any{map[string]any{"role": "user", "content": "hello"}}, + "output": map[string]any{"format": "text", "validation_mode": "none"}, + }}}, + })}, + }}) + if err != nil { + t.Fatal(err) + } + inspected, err := engine.InspectProfile(context.Background(), " literal ") + if err != nil { + t.Fatal(err) + } + prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"}) + if err != nil { + t.Fatal(err) + } + if inspected.ProfileID != " literal " || prepared.SelectedProfileID != inspected.ProfileID || prepared.EffectiveModelParams.Model != inspected.EffectiveModelParams.Model { + t.Fatalf("inspection=%+v preparation=%+v", inspected, prepared) + } + if _, err := engine.InspectProfile(context.Background(), "literal"); !errors.Is(err, promptkit.ErrProfileNotFound) { + t.Fatalf("trimmed ID resolved unexpectedly: %v", err) + } +} + +func TestInspectProfileIsIndependentAndDoesNotInvokeProvider(t *testing.T) { + client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}} + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{ + Source: configkit.Mapping("inspection", map[string]any{"profiles": map[string]any{ + "profile": map[string]any{ + "endpoint": "https://example.test/v1", "model": "model", "api_key_required": true, + "extra_params": map[string]any{"nested": map[string]any{"value": "original"}}, + }, + }}), + }}}, promptkit.WithLLMClient(client)) + if err != nil { + t.Fatal(err) + } + first, err := engine.InspectProfile(context.Background(), "profile") + if err != nil || first == nil || !first.APIKeyRequired { + t.Fatalf("first inspection=(%+v, %v)", first, err) + } + first.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["value"] = "changed" + second, err := engine.InspectProfile(context.Background(), "profile") + if err != nil || second == nil || second.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["value"] != "original" || len(client.requests) != 0 { + t.Fatalf("inspection changed snapshot or invoked provider: (%+v, %v), calls=%d", second, err, len(client.requests)) + } +} + +func TestInspectProfilePublicErrorsAndPartialTarget(t *testing.T) { + var nilEngine *promptkit.Engine + if _, err := nilEngine.InspectProfile(context.Background(), "profile"); !errors.Is(err, promptkit.ErrInvalidConfig) { + t.Fatalf("nil engine: %v", err) + } + engine, err := promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{ + Source: configkit.Mapping("partial", map[string]any{"profiles": map[string]any{"profile": map[string]any{"model": "model"}}}), + }}}) + if err != nil { + t.Fatal(err) + } + if _, err := engine.InspectProfile(context.Background(), " "); !errors.Is(err, promptkit.ErrInvalidRequest) { + t.Fatalf("blank selection: %v", err) + } + if _, err := engine.InspectProfile(context.Background(), "missing"); !errors.Is(err, promptkit.ErrProfileNotFound) { + t.Fatalf("missing selection: %v", err) + } + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := engine.InspectProfile(ctx, "profile"); !errors.Is(err, context.Canceled) { + t.Fatalf("canceled inspection: %v", err) + } + partial, err := engine.InspectProfile(context.Background(), "profile") + if err != nil || partial == nil || partial.EffectiveModelParams.Endpoint != "" || partial.EffectiveModelParams.Model != "model" { + t.Fatalf("partial inspection=(%+v, %v)", partial, err) + } +} diff --git a/public_contract_test.go b/public_contract_test.go deleted file mode 100644 index cdda81a..0000000 --- a/public_contract_test.go +++ /dev/null @@ -1,1497 +0,0 @@ -package promptkit_test - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "reflect" - "strings" - "sync" - "sync/atomic" - "testing" - "testing/fstest" - - "gitea.maximumdirect.net/eric/promptkit" - "gitea.maximumdirect.net/eric/promptkit/testfixture" -) - -func TestInspectProfileResolvesCredentialStatesWithoutPromptOrGeneration(t *testing.T) { - const environmentName = "PROMPTKIT_INSPECTION_ABSENT_KEY" - t.Setenv(environmentName, "") - client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}} - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(fstest.MapFS{}, "."), - testfixture.WithProfileFS(fstest.MapFS{ - "environment.yaml": &fstest.MapFile{Data: []byte(`id: environment -endpoint: http://environment.example/v1 -model: environment-model -api_key_env: PROMPTKIT_INSPECTION_ABSENT_KEY -`)}, - }, "."), - testfixture.WithProfiles( - testfixture.Profile{ID: "direct", Endpoint: "http://direct.example/v1", Model: "direct-model", APIKeyRequired: true}, - testfixture.Profile{ID: "none", Endpoint: "http://none.example/v1", Model: "none-model"}, - ), - promptkit.WithLLMClient(client), - ) - if err != nil { - t.Fatalf("construct inspection engine: %v", err) - } - - for _, tc := range []struct { - profileID string - wantEnv string - wantDirectKey bool - wantEndpoint string - }{ - {profileID: "environment", wantEnv: environmentName, wantEndpoint: "http://environment.example/v1"}, - {profileID: "direct", wantDirectKey: true, wantEndpoint: "http://direct.example/v1"}, - {profileID: "none", wantEndpoint: "http://none.example/v1"}, - } { - t.Run(tc.profileID, func(t *testing.T) { - inspection, err := engine.InspectProfile(context.Background(), tc.profileID) - if err != nil { - t.Fatalf("inspect profile: %v", err) - } - if inspection.ProfileID != tc.profileID || - inspection.EffectiveModelParams.Endpoint != tc.wantEndpoint || - inspection.EffectiveModelParams.BackendID != "" || - inspection.EffectiveModelParams.APIKeyEnv != tc.wantEnv || - inspection.APIKeyRequired != tc.wantDirectKey { - t.Fatalf("unexpected inspection: %#v", inspection) - } - }) - } - if len(client.requests) != 0 { - t.Fatalf("inspection invoked the model client %d times", len(client.requests)) - } -} - -func TestFileProfileExactIDMatchesInspectionAndPreparation(t *testing.T) { - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prompt", " normalized-profile ", "message"), "."), - testfixture.WithProfileFS(fstest.MapFS{ - "profile.yaml": &fstest.MapFile{Data: []byte(` -id: " normalized-profile " -endpoint: http://profile.example/v1 -model: normalized-model -`)}, - }, "."), - ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - - inspection, err := engine.InspectProfile(context.Background(), " normalized-profile ") - if err != nil { - t.Fatalf("inspect normalized profile: %v", err) - } - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: "prompt", - ProfileID: " normalized-profile ", - }) - if err != nil { - t.Fatalf("prepare with normalized profile: %v", err) - } - - if inspection.ProfileID != " normalized-profile " || - prepared.SelectedProfileID != inspection.ProfileID || - inspection.EffectiveModelParams.Model != "normalized-model" || - prepared.EffectiveModelParams.Model != inspection.EffectiveModelParams.Model { - t.Fatalf("inspection=%#v prepared=%#v", inspection, prepared) - } -} - -func TestInspectProfilePreservesPublicErrorIdentities(t *testing.T) { - newEngine := func(t *testing.T, options ...any) *promptkit.Engine { - t.Helper() - engine, err := testfixture.NewEngine( - testfixture.Config{}, - append([]any{testfixture.WithPromptFS(fstest.MapFS{}, ".")}, options...)..., - ) - if err != nil { - t.Fatalf("construct inspection engine: %v", err) - } - return engine - } - - var nilEngine *promptkit.Engine - if result, err := nilEngine.InspectProfile(context.Background(), "profile"); result != nil || - !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("nil engine result=(%#v, %v), want ErrInvalidConfig", result, err) - } - - valid := newEngine(t, testfixture.WithProfiles(testfixture.Profile{ - ID: "profile", Endpoint: "http://profile.example/v1", Model: "model", - })) - if result, err := valid.InspectProfile(context.Background(), " \t "); result != nil || - !errors.Is(err, promptkit.ErrInvalidRequest) { - t.Fatalf("blank profile result=(%#v, %v), want ErrInvalidRequest", result, err) - } - if result, err := valid.InspectProfile(context.Background(), "missing"); result != nil || - !errors.Is(err, promptkit.ErrProfileNotFound) || errors.Is(err, promptkit.ErrProfileLoad) { - t.Fatalf("missing profile result=(%#v, %v), want only ErrProfileNotFound", result, err) - } - - malformed, err := testfixture.NewEngine(testfixture.Config{}, testfixture.WithProfileFS(fstest.MapFS{ - "broken.yaml": &fstest.MapFile{Data: []byte("id: broken\nendpoint: http://broken.example/v1\nmodel: model\nextra_params:\n invalid: .nan\n")}, - }, ".")) - if malformed != nil || !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("malformed profile construction=(%#v, %v), want ErrInvalidConfig", malformed, err) - } - - unknownBackend, err := testfixture.NewEngine(testfixture.Config{}, testfixture.WithProfiles(testfixture.Profile{ - ID: "unknown-backend", BackendID: "unknown", Model: "model", - })) - if unknownBackend != nil || !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("unknown backend construction=(%#v, %v), want ErrInvalidConfig", unknownBackend, err) - } - - canceled := newEngine(t, testfixture.WithProfiles(testfixture.Profile{ID: "profile", Endpoint: "http://profile.example/v1"})) - ctx, cancel := context.WithCancel(context.Background()) - cancel() - if result, err := canceled.InspectProfile(ctx, "profile"); result != nil || !errors.Is(err, context.Canceled) { - t.Fatalf("canceled inspection result=(%#v, %v)", result, err) - } -} - -func TestInspectProfileReturnsIndependentTargetMatchingPreparation(t *testing.T) { - extraParams := map[string]any{ - "nested": map[string]any{"value": "original"}, - } - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), - testfixture.WithProfiles(testfixture.Profile{ - ID: "profile", Endpoint: "http://profile.example/v1", Model: "model", ExtraParams: extraParams, - }), - ) - if err != nil { - t.Fatalf("construct inspection engine: %v", err) - } - - first, err := engine.InspectProfile(context.Background(), "profile") - if err != nil { - t.Fatalf("first inspection: %v", err) - } - first.EffectiveModelParams.ExtraParams["nested"].(map[string]any)["value"] = "changed" - first.EffectiveModelParams.ExtraParams["later"] = true - - second, err := engine.InspectProfile(context.Background(), "profile") - if err != nil { - t.Fatalf("second inspection: %v", err) - } - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"}) - if err != nil { - t.Fatalf("prepare after inspection mutation: %v", err) - } - for _, target := range []promptkit.ExecutionTarget{second.EffectiveModelParams, prepared.EffectiveModelParams} { - if target.ExtraParams["nested"].(map[string]any)["value"] != "original" || target.ExtraParams["later"] != nil { - t.Fatalf("inspection mutation reached engine-owned target: %#v", target.ExtraParams) - } - } - if !reflect.DeepEqual(second.EffectiveModelParams, prepared.EffectiveModelParams) { - t.Fatalf("inspection target=%#v, preparation target=%#v", second.EffectiveModelParams, prepared.EffectiveModelParams) - } -} - -func TestInspectPromptReturnsDeclaredMetadataWithoutExecutionWork(t *testing.T) { - client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "unexpected"}} - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(fstest.MapFS{ - "report-v1.yaml": &fstest.MapFile{Data: []byte(`id: report -version: "1.0.0" -messages: - - role: user - content: old report -output: - format: text - validation_mode: none -`)}, - "report-v2.yaml": &fstest.MapFile{Data: []byte(`id: report -version: "2.0.0" -default_profile: missing-profile -inputs: - - name: location - required: true - content_type: text/plain - description: Forecast location. - - name: units - content_type: text/plain - description: Unit preference. -messages: - - role: user - content_file: messages/report.md -output: - format: json - validation_mode: json_schema - schema_path: schemas/report.json -`)}, - "messages/report.md": &fstest.MapFile{Data: []byte("rendered report body is not returned")}, - }, "."), - promptkit.WithLLMClient(client), - ) - if err != nil { - t.Fatalf("construct prompt-inspection engine: %v", err) - } - - inspection, err := engine.InspectPrompt(context.Background(), "report", "2.0.0") - if err != nil { - t.Fatalf("inspect prompt: %v", err) - } - if inspection.PromptID != "report" || - inspection.PromptVersion != "2.0.0" || - inspection.PromptHash == "" || - inspection.DefaultProfileID != "missing-profile" || - inspection.OutputContract != (promptkit.OutputContract{ - Format: promptkit.FormatJSON, - ValidationMode: promptkit.ValidationJSONSchema, - SchemaPath: "schemas/report.json", - }) { - t.Fatalf("unexpected inspection metadata: %#v", inspection) - } - wantInputs := []promptkit.PromptInputDefinition{ - {Name: "location", Required: true, ContentType: "text/plain", Description: "Forecast location."}, - {Name: "units", ContentType: "text/plain", Description: "Unit preference."}, - } - if !reflect.DeepEqual(inspection.Inputs, wantInputs) { - t.Fatalf("inspection inputs=%#v, want %#v", inspection.Inputs, wantInputs) - } - if len(client.requests) != 0 { - t.Fatalf("inspection invoked the model client %d times", len(client.requests)) - } -} - -func TestInspectPromptPreservesPublicErrorIdentities(t *testing.T) { - newEngine := func(t *testing.T, source fstest.MapFS) *promptkit.Engine { - t.Helper() - engine, err := testfixture.NewEngine(testfixture.Config{}, testfixture.WithPromptFS(source, ".")) - if err != nil { - t.Fatalf("construct prompt-inspection engine: %v", err) - } - return engine - } - validSource := fstest.MapFS{ - "prompt.yaml": &fstest.MapFile{Data: []byte(`id: prompt -version: "1" -messages: - - role: user - content: body -output: - format: text - validation_mode: none -`)}, - } - - var nilEngine *promptkit.Engine - if result, err := nilEngine.InspectPrompt(context.Background(), "prompt", "1"); result != nil || - !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("nil engine result=(%#v, %v), want ErrInvalidConfig", result, err) - } - - valid := newEngine(t, validSource) - if result, err := valid.InspectPrompt(context.Background(), " \t ", "1"); result != nil || - !errors.Is(err, promptkit.ErrInvalidRequest) { - t.Fatalf("blank prompt result=(%#v, %v), want ErrInvalidRequest", result, err) - } - if result, err := valid.InspectPrompt(context.Background(), "missing", "1"); result != nil || - !errors.Is(err, promptkit.ErrPromptNotFound) || errors.Is(err, promptkit.ErrPromptLoad) { - t.Fatalf("missing prompt result=(%#v, %v), want only ErrPromptNotFound", result, err) - } - if result, err := valid.InspectPrompt(context.Background(), "prompt", "missing"); result != nil || - !errors.Is(err, promptkit.ErrPromptNotFound) || errors.Is(err, promptkit.ErrPromptLoad) { - t.Fatalf("missing version result=(%#v, %v), want only ErrPromptNotFound", result, err) - } - - ambiguous := newEngine(t, fstest.MapFS{ - "one.yaml": &fstest.MapFile{Data: []byte(`id: prompt -version: "1" -messages: - - role: user - content: first -output: - format: text - validation_mode: none -`)}, - "two.yaml": &fstest.MapFile{Data: []byte(`id: prompt -version: "2" -messages: - - role: user - content: second -output: - format: text - validation_mode: none -`)}, - }) - if result, err := ambiguous.InspectPrompt(context.Background(), "prompt", ""); result != nil || - !errors.Is(err, promptkit.ErrPromptLoad) { - t.Fatalf("ambiguous prompt result=(%#v, %v), want ErrPromptLoad", result, err) - } - - for name, source := range map[string]fstest.MapFS{ - "malformed definition": { - "broken.yaml": &fstest.MapFile{Data: []byte("id: broken\nversion: \"1\"\nunknown: value\n")}, - }, - "missing content file": { - "broken.yaml": &fstest.MapFile{Data: []byte(`id: broken -version: "1" -messages: - - role: user - content_file: missing.md -output: - format: text - validation_mode: none -`)}, - }, - } { - t.Run(name, func(t *testing.T) { - engine, err := testfixture.NewEngine(testfixture.Config{}, testfixture.WithPromptFS(source, ".")) - if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("broken prompt construction=(%#v, %v), want ErrInvalidConfig", engine, err) - } - }) - } - - canceled := valid - ctx, cancel := context.WithCancel(context.Background()) - cancel() - if result, err := canceled.InspectPrompt(ctx, "prompt", "1"); result != nil || - !errors.Is(err, context.Canceled) { - t.Fatalf("canceled inspection result=(%#v, %v)", result, err) - } -} - -func TestInspectPromptReturnsIndependentMetadataMatchingPreparation(t *testing.T) { - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(fstest.MapFS{ - "prompt.yaml": &fstest.MapFile{Data: []byte(`id: prompt -version: "1" -default_profile: profile -inputs: - - name: subject - content_type: text/plain - description: Summary subject. -messages: - - role: user - content: summarize -output: - format: markdown - validation_mode: basic -`)}, - }, "."), - testfixture.WithProfiles(testfixture.Profile{ - ID: "profile", Endpoint: "http://profile.example/v1", Model: "model", - }), - ) - if err != nil { - t.Fatalf("construct prompt-inspection engine: %v", err) - } - - first, err := engine.InspectPrompt(context.Background(), "prompt", "1") - if err != nil { - t.Fatalf("first inspection: %v", err) - } - first.Inputs[0].Name = "changed" - first.OutputContract.SchemaPath = "changed.json" - - second, err := engine.InspectPrompt(context.Background(), "prompt", "1") - if err != nil { - t.Fatalf("second inspection: %v", err) - } - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt", PromptVersion: "1"}) - if err != nil { - t.Fatalf("prepare after inspection mutation: %v", err) - } - if second.Inputs[0].Name != "subject" || second.OutputContract.SchemaPath != "" || - prepared.OutputContract.SchemaPath != "" || second.PromptHash != prepared.PromptHash { - t.Fatalf("inspection mutation reached engine-owned prompt metadata: inspection=%#v prepared=%#v", second, prepared) - } -} - -func TestBackendIdentityJSONNamesAndOmission(t *testing.T) { - t.Run("execution target round trip", func(t *testing.T) { - value := promptkit.ExecutionTarget{BackendID: promptkit.BackendOpenRouter} - payload, err := json.Marshal(value) - if err != nil { - t.Fatalf("marshal execution target: %v", err) - } - var decoded promptkit.ExecutionTarget - if err := json.Unmarshal(payload, &decoded); err != nil { - t.Fatalf("unmarshal execution target: %v", err) - } - if decoded.BackendID != value.BackendID { - t.Fatalf("backend identity did not round trip: got %q want %q", decoded.BackendID, value.BackendID) - } - }) - - t.Run("prepared run round trip", func(t *testing.T) { - value := promptkit.PreparedRun{SelectedBackendID: promptkit.BackendOpenRouter} - payload, err := json.Marshal(value) - if err != nil { - t.Fatalf("marshal prepared run: %v", err) - } - var decoded promptkit.PreparedRun - if err := json.Unmarshal(payload, &decoded); err != nil { - t.Fatalf("unmarshal prepared run: %v", err) - } - if decoded.SelectedBackendID != value.SelectedBackendID { - t.Fatalf("backend identity did not round trip: got %q want %q", decoded.SelectedBackendID, value.SelectedBackendID) - } - }) - - t.Run("run result round trip", func(t *testing.T) { - value := promptkit.RunResult{SelectedBackendID: promptkit.BackendOpenRouter} - payload, err := json.Marshal(value) - if err != nil { - t.Fatalf("marshal run result: %v", err) - } - var decoded promptkit.RunResult - if err := json.Unmarshal(payload, &decoded); err != nil { - t.Fatalf("unmarshal run result: %v", err) - } - if decoded.SelectedBackendID != value.SelectedBackendID { - t.Fatalf("backend identity did not round trip: got %q want %q", decoded.SelectedBackendID, value.SelectedBackendID) - } - }) - - payload, err := json.Marshal(promptkit.ExecutionTarget{}) - if err != nil { - t.Fatalf("marshal empty execution target: %v", err) - } - if strings.Contains(string(payload), `"backend_id"`) { - t.Fatalf("empty backend identity was not omitted: %s", payload) - } -} - -func TestEndpointOnlyProfileOmitsBackendIdentityFromStableJSON(t *testing.T) { - client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}} - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), - testfixture.WithProfiles(testfixture.Profile{ - ID: "profile", Endpoint: "http://example.test/v1", Model: "model", - }), - promptkit.WithLLMClient(client), - ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"}) - if err != nil { - t.Fatalf("prepare endpoint-only profile: %v", err) - } - result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"}) - if err != nil { - t.Fatalf("run endpoint-only profile: %v", err) - } - if prepared.SelectedBackendID != "" || - prepared.EffectiveModelParams.BackendID != "" || - result.SelectedBackendID != "" || - result.EffectiveModelParams.BackendID != "" { - t.Fatalf("endpoint-only profile acquired backend identity: prepared=%+v result=%+v", prepared, result) - } - for _, value := range []any{prepared, result} { - payload, err := json.Marshal(value) - if err != nil { - t.Fatalf("marshal endpoint-only value: %v", err) - } - if strings.Contains(string(payload), `"backend_id"`) || strings.Contains(string(payload), `"selected_backend_id"`) { - t.Fatalf("endpoint-only backend identity was not omitted: %s", payload) - } - } -} - -func TestUnknownProfileBackendFailsConstruction(t *testing.T) { - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), - testfixture.WithProfiles(testfixture.Profile{ID: "profile", BackendID: "unknown", Model: "model"}), - ) - if engine != nil || !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("construction=(%#v, %v), want ErrInvalidConfig", engine, err) - } -} - -func TestLocalBackendConstructsAndRegistersConventionalBackend(t *testing.T) { - const ( - localBackendID = "local" - limit = 2 - ) - if promptkit.BackendLocal != localBackendID { - t.Fatalf("BackendLocal=%q, want %q", promptkit.BackendLocal, localBackendID) - } - - endpoint := "http://local.example/v1" - backend := testfixture.LocalBackend(endpoint, limit) - want := testfixture.Backend{ - ID: localBackendID, - Endpoint: endpoint, - ConcurrencyLimit: limit, - } - if !reflect.DeepEqual(backend, want) { - t.Fatalf("LocalBackend()=%+v, want %+v", backend, want) - } - - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prompt", "local-profile", "message"), "."), - testfixture.WithBackend(backend), - testfixture.WithProfiles(testfixture.Profile{ - ID: "local-profile", - BackendID: localBackendID, - Model: "model", - }), - ) - if err != nil { - t.Fatalf("construct engine with local backend: %v", err) - } - - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"}) - if err != nil { - t.Fatalf("prepare with local backend: %v", err) - } - if prepared.SelectedBackendID != localBackendID || - prepared.EffectiveModelParams.Endpoint != endpoint { - t.Fatalf("unexpected local backend preparation: %+v", prepared) - } -} - -func TestCustomBackendFlowsThroughProfilesOverridesAndInjectedClient(t *testing.T) { - t.Setenv("CUSTOM_LLM_KEY", "test-key") - client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}} - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prompt", "backend-profile", "message"), "."), - testfixture.WithBackend(testfixture.Backend{ - ID: "custom", - Endpoint: " http://backend.example/v1 ", - APIKeyEnv: " CUSTOM_LLM_KEY ", - ExtraParams: map[string]any{ - "provider": "custom", - }, - }), - testfixture.WithProfiles( - testfixture.Profile{ID: "backend-profile", BackendID: "custom", Model: "backend-model"}, - testfixture.Profile{ID: "profile-endpoint", BackendID: "custom", Endpoint: "http://profile.example/v1", Model: "profile-model"}, - testfixture.Profile{ID: "blank-profile-endpoint", BackendID: "custom", Model: "profile-model"}, - ), - promptkit.WithLLMClient(client), - ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - - result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"}) - if err != nil { - t.Fatalf("run with custom backend: %v", err) - } - if len(client.requests) != 1 { - t.Fatalf("expected one injected-client request, got %d", len(client.requests)) - } - target := client.requests[0].Target - if target.BackendID != "custom" || - target.Endpoint != "http://backend.example/v1" || - target.APIKeyEnv != "CUSTOM_LLM_KEY" || - target.Model != "backend-model" || - target.ExtraParams["provider"] != "custom" || - result.SelectedBackendID != "custom" { - t.Fatalf("unexpected custom backend settings: target=%+v result_backend=%q", target, result.SelectedBackendID) - } - - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: "prompt", ProfileID: "profile-endpoint", - }) - if err != nil { - t.Fatalf("prepare profile endpoint override: %v", err) - } - if prepared.SelectedBackendID != "custom" || prepared.EffectiveModelParams.Endpoint != "http://profile.example/v1" { - t.Fatalf("profile endpoint override changed backend identity: %+v", prepared) - } - - prepared, err = engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: "prompt", ProfileID: "blank-profile-endpoint", - }) - if err != nil { - t.Fatalf("prepare blank profile endpoint: %v", err) - } - if prepared.SelectedBackendID != "custom" || prepared.EffectiveModelParams.Endpoint != "http://backend.example/v1" { - t.Fatalf("blank profile endpoint did not inherit backend endpoint: %+v", prepared) - } - - prepared, err = engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: "prompt", - Execution: &promptkit.ExecutionTargetOverride{ - Endpoint: "http://request.example/v1", - }, - }) - if err != nil { - t.Fatalf("prepare request endpoint override: %v", err) - } - if prepared.SelectedBackendID != "custom" || prepared.EffectiveModelParams.Endpoint != "http://request.example/v1" { - t.Fatalf("request endpoint override changed backend identity: %+v", prepared) - } -} - -func TestCustomBackendSupportsFileProfileAndBothSelectionPaths(t *testing.T) { - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prompt", "file-profile", "message"), "."), - testfixture.WithProfileFS(fstest.MapFS{ - "profile.yaml": &fstest.MapFile{Data: []byte(`id: file-profile -backend: file-backend -model: file-model -`)}, - }, "."), - testfixture.WithBackend(testfixture.Backend{ - ID: "file-backend", - Endpoint: "http://file-backend.example/v1", - }), - ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - - for _, request := range []promptkit.RunRequest{ - {PromptID: "prompt"}, - {PromptID: "prompt", ProfileID: "file-profile"}, - } { - prepared, err := engine.Prepare(context.Background(), request) - if err != nil { - t.Fatalf("prepare file profile: %v", err) - } - if prepared.SelectedBackendID != "file-backend" || - prepared.EffectiveModelParams.Endpoint != "http://file-backend.example/v1" { - t.Fatalf("unexpected file-profile backend resolution: %+v", prepared) - } - } -} - -func TestBackendOptionsAccumulateAndRegistrationsAreEngineLocal(t *testing.T) { - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prompt", "first-profile", "message"), "."), - testfixture.WithBackend(testfixture.Backend{ID: "first", Endpoint: "http://first.example/v1"}), - testfixture.WithBackend(testfixture.Backend{ID: "second", Endpoint: "http://second.example/v1"}), - testfixture.WithProfiles( - testfixture.Profile{ID: "first-profile", BackendID: "first", Model: "model"}, - testfixture.Profile{ID: "second-profile", BackendID: "second", Model: "model"}, - ), - ) - if err != nil { - t.Fatalf("construct engine with accumulated registrations: %v", err) - } - for profileID, wantEndpoint := range map[string]string{ - "first-profile": "http://first.example/v1", - "second-profile": "http://second.example/v1", - } { - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: "prompt", ProfileID: profileID, - }) - if err != nil { - t.Fatalf("prepare %s: %v", profileID, err) - } - if prepared.EffectiveModelParams.Endpoint != wantEndpoint { - t.Fatalf("profile %s endpoint=%q, want %q", profileID, prepared.EffectiveModelParams.Endpoint, wantEndpoint) - } - } - - newEngine := func(endpoint string) *promptkit.Engine { - t.Helper() - value, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), - testfixture.WithBackend(testfixture.Backend{ID: "same-id", Endpoint: endpoint}), - testfixture.WithProfiles(testfixture.Profile{ID: "profile", BackendID: "same-id", Model: "model"}), - ) - if err != nil { - t.Fatalf("construct isolated engine: %v", err) - } - return value - } - firstEngine := newEngine("http://one.example/v1") - secondEngine := newEngine("http://two.example/v1") - for engine, wantEndpoint := range map[*promptkit.Engine]string{ - firstEngine: "http://one.example/v1", - secondEngine: "http://two.example/v1", - } { - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"}) - if err != nil { - t.Fatalf("prepare isolated engine: %v", err) - } - if prepared.EffectiveModelParams.Endpoint != wantEndpoint { - t.Fatalf("isolated engine endpoint=%q, want %q", prepared.EffectiveModelParams.Endpoint, wantEndpoint) - } - } -} - -func TestWithBackendCopiesQueueCapacity(t *testing.T) { - queueCapacity := 4 - option := testfixture.WithBackend(testfixture.Backend{ - ID: "custom", - Endpoint: "http://custom.example/v1", - ConcurrencyLimit: 1, - QueueCapacity: &queueCapacity, - }) - queueCapacity = -1 - - _, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), - option, - testfixture.WithProfiles(testfixture.Profile{ - ID: "profile", BackendID: "custom", Model: "model", - }), - ) - if err != nil { - t.Fatalf("construct engine after mutating queue pointer: %v", err) - } -} - -func TestBackendConfigRejectsInvalidDefinitions(t *testing.T) { - cycle := map[string]any{} - cycle["self"] = cycle - tests := []struct { - name string - backends []testfixture.Backend - }{ - {name: "blank id", backends: []testfixture.Backend{{Endpoint: "http://example.test/v1"}}}, - {name: "invalid endpoint", backends: []testfixture.Backend{{ID: "custom", Endpoint: "ftp://example.test/v1"}}}, - {name: "invalid environment", backends: []testfixture.Backend{{ID: "custom", Endpoint: "http://example.test/v1", APIKeyEnv: "BAD-NAME"}}}, - {name: "reserved extra parameter", backends: []testfixture.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: map[string]any{"model": "override"}}}}, - {name: "cyclic extra parameter", backends: []testfixture.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: cycle}}}, - {name: "malformed JSON number", backends: []testfixture.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: map[string]any{"value": json.Number("01")}}}}, - {name: "excessively deep extra parameter", backends: []testfixture.Backend{{ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: map[string]any{"value": excessivelyDeepJSONValue()}}}}, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - options := []any{ - testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), - } - for _, backend := range tt.backends { - options = append(options, testfixture.WithBackend(backend)) - } - _, err := testfixture.NewEngine(testfixture.Config{}, options...) - if !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("expected ErrInvalidConfig, got %v", err) - } - }) - } -} - -func TestBackendExtraParamsAreDeeplyCopiedAtConstructionAndLookup(t *testing.T) { - nested := map[string]any{"value": "original"} - extraParams := map[string]any{"nested": nested} - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), - testfixture.WithBackend(testfixture.Backend{ - ID: "custom", Endpoint: "http://example.test/v1", ExtraParams: extraParams, - }), - testfixture.WithProfiles(testfixture.Profile{ID: "profile", BackendID: "custom", Model: "model"}), - ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - nested["value"] = "mutated input" - extraParams["later"] = true - - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"}) - if err != nil { - t.Fatalf("first prepare: %v", err) - } - gotNested := prepared.EffectiveModelParams.ExtraParams["nested"].(map[string]any) - if gotNested["value"] != "original" || prepared.EffectiveModelParams.ExtraParams["later"] != nil { - t.Fatalf("backend retained caller mutations: %#v", prepared.EffectiveModelParams.ExtraParams) - } - gotNested["value"] = "mutated lookup" - - prepared, err = engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"}) - if err != nil { - t.Fatalf("second prepare: %v", err) - } - gotNested = prepared.EffectiveModelParams.ExtraParams["nested"].(map[string]any) - if gotNested["value"] != "original" { - t.Fatalf("backend retained lookup mutation: %#v", prepared.EffectiveModelParams.ExtraParams) - } -} - -func TestEngineValidationWithZeroRepairBudgetIsSinglePass(t *testing.T) { - client := &fakeLLMClient{ - response: &promptkit.GenerateResponse{Content: "not-json"}, - } - engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(client)) - - result, err := engine.Run(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - Validation: &promptkit.OutputContract{ - Format: promptkit.FormatJSON, - ValidationMode: promptkit.ValidationJSON, - RepairAttempts: 0, - }, - }) - if err != nil { - t.Fatalf("run with failed content validation: %v", err) - } - if result.Validation.Status != promptkit.ValidationFailed || - result.Validation.RepairAttempts != 0 { - t.Fatalf("expected failed single-pass validation, got %#v", result.Validation) - } - if len(client.requests) != 1 { - t.Fatalf("expected one model generation, got %d", len(client.requests)) - } -} - -func TestEngineRunRepairsJSONSchemaOutput(t *testing.T) { - client := &fakeLLMClient{responses: []*promptkit.GenerateResponse{ - { - Content: "{}", - Usage: promptkit.TokenUsage{PromptTokens: 2, CompletionTokens: 3, TotalTokens: 5, CachedTokens: 7, CacheWriteTokens: 11}, - }, - { - Content: `{"events":[{"title":"Repaired event"}]}`, - Usage: promptkit.TokenUsage{PromptTokens: 13, CompletionTokens: 17, TotalTokens: 19, CachedTokens: 23, CacheWriteTokens: 29}, - }, - }} - engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(client)) - - result, err := engine.Run(context.Background(), promptkit.RunRequest{ - PromptID: frameworkStructuredEventsPromptID, - SessionID: " repair-session ", - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - Validation: &promptkit.OutputContract{ - Format: promptkit.FormatJSON, - ValidationMode: promptkit.ValidationJSONSchema, - SchemaPath: "structured_events.schema.json", - RepairAttempts: 1, - }, - }) - if err != nil { - t.Fatalf("run: %v", err) - } - if result.RawOutput != client.responses[1].Content || result.Validation.Status != promptkit.ValidationPassed || - result.Validation.RepairAttempts != 1 { - t.Fatalf("repaired result = %+v", result) - } - wantUsage := promptkit.TokenUsage{PromptTokens: 15, CompletionTokens: 20, TotalTokens: 24, CachedTokens: 30, CacheWriteTokens: 40} - if result.Usage != wantUsage { - t.Fatalf("usage = %+v, want %+v", result.Usage, wantUsage) - } - if len(client.requests) != 2 { - t.Fatalf("generation calls = %d, want 2", len(client.requests)) - } - initial, repaired := client.requests[0], client.requests[1] - if initial.Prompt.SessionID != "repair-session" || repaired.Prompt.SessionID != initial.Prompt.SessionID || - !reflect.DeepEqual(repaired.Target, initial.Target) || repaired.TargetPresence != initial.TargetPresence || - !reflect.DeepEqual(repaired.StructuredOutput, initial.StructuredOutput) { - t.Fatalf("generation request state drifted: initial=%+v repaired=%+v", initial, repaired) - } - if initial.StructuredOutput == nil || initial.StructuredOutput.JSONSchema == nil { - t.Fatalf("expected structured output on initial request: %+v", initial) - } -} - -func TestEngineRunReturnsFinalResultAfterRepairExhaustion(t *testing.T) { - client := &fakeLLMClient{responses: []*promptkit.GenerateResponse{ - {Content: "not-json", Usage: promptkit.TokenUsage{PromptTokens: 2, CompletionTokens: 3, TotalTokens: 5}}, - {Content: "still-not-json", Usage: promptkit.TokenUsage{PromptTokens: 7, CompletionTokens: 11, TotalTokens: 18}}, - }} - engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(client)) - - result, err := engine.Run(context.Background(), promptkit.RunRequest{ - PromptID: frameworkMarkdownSummaryPromptID, - Inputs: map[string]promptkit.ArtifactRef{ - "transcript": promptkit.Inline("Rin opens the gate."), - "glossary": promptkit.Inline("gate: A guarded passage."), - }, - Validation: &promptkit.OutputContract{ - Format: promptkit.FormatJSON, - ValidationMode: promptkit.ValidationJSON, - RepairAttempts: 1, - }, - }) - if err != nil || result == nil { - t.Fatalf("run = (%+v, %v), want exhausted result", result, err) - } - if result.RawOutput != "still-not-json" || result.Validation.Status != promptkit.ValidationFailed || - result.Validation.RepairAttempts != 1 || len(result.Validation.Errors) == 0 || - result.Usage != (promptkit.TokenUsage{PromptTokens: 9, CompletionTokens: 14, TotalTokens: 23}) { - t.Fatalf("exhausted result = %+v", result) - } -} - -func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) { - profile := testfixture.Profile{ID: "profile", Endpoint: "http://example.test/v1", Model: "model"} - - t.Run("prompt source", func(t *testing.T) { - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("first", "profile", "first"), "."), - testfixture.WithPromptFS(contractPromptFS("second", "profile", "second"), "."), - testfixture.WithProfiles(profile), - ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "second"}) - if err != nil { - t.Fatalf("prepare from last prompt source: %v", err) - } - if prepared.Messages[0].Content != "second" { - t.Fatalf("expected last prompt source, got %#v", prepared.Messages) - } - }) - - t.Run("profile source", func(t *testing.T) { - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), - testfixture.WithProfileFS(contractProfileFS("profile", "first-model"), "."), - testfixture.WithProfileFS(contractProfileFS("profile", "second-model"), "."), - ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"}) - if err != nil { - t.Fatalf("prepare from last profile source: %v", err) - } - if prepared.EffectiveModelParams.Model != "second-model" { - t.Fatalf("expected last profile source, got %q", prepared.EffectiveModelParams.Model) - } - }) - - t.Run("fallback profile source", func(t *testing.T) { - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), - testfixture.WithFallbackProfileFS(contractProfileFS("profile", "first-model"), "."), - testfixture.WithFallbackProfileFS(contractProfileFS("profile", "second-model"), "."), - ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"}) - if err != nil { - t.Fatalf("prepare from last fallback profile source: %v", err) - } - if prepared.EffectiveModelParams.Model != "second-model" { - t.Fatalf("expected last fallback profile source, got %q", prepared.EffectiveModelParams.Model) - } - }) - - t.Run("in-memory profiles", func(t *testing.T) { - first := profile - first.Model = "first-model" - second := profile - second.Model = "second-model" - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), - testfixture.WithProfiles(first), - testfixture.WithProfiles(second), - ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"}) - if err != nil { - t.Fatalf("prepare from last in-memory profile option: %v", err) - } - if prepared.EffectiveModelParams.Model != "second-model" { - t.Fatalf("expected last in-memory profiles, got %q", prepared.EffectiveModelParams.Model) - } - }) - - t.Run("schema source", func(t *testing.T) { - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractSchemaPromptFS(), "."), - testfixture.WithProfiles(profile), - promptkit.WithSchemaFS(contractSchemaFS("first"), "."), - promptkit.WithSchemaFS(contractSchemaFS("second"), "."), - ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "schema-prompt"}) - if err != nil { - t.Fatalf("prepare from last schema source: %v", err) - } - schema := prepared.StructuredOutput.JSONSchema.Schema.(map[string]any) - if schema["title"] != "second" { - t.Fatalf("expected last schema source, got %#v", schema) - } - }) - - t.Run("model client", func(t *testing.T) { - var firstCalls, secondCalls atomic.Int64 - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), - testfixture.WithProfiles(profile), - promptkit.WithLLMClient(countingLLMClient{calls: &firstCalls}), - promptkit.WithLLMClient(countingLLMClient{calls: &secondCalls}), - ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - if _, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"}); err != nil { - t.Fatalf("run with last model client: %v", err) - } - if firstCalls.Load() != 0 || secondCalls.Load() != 1 { - t.Fatalf("expected only last client call, got first=%d second=%d", firstCalls.Load(), secondCalls.Load()) - } - }) - - t.Run("artifact reader", func(t *testing.T) { - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractInputPromptFS(), "."), - testfixture.WithProfiles(profile), - promptkit.WithArtifactReader(fixedArtifactReader("first")), - promptkit.WithArtifactReader(fixedArtifactReader("second")), - ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: "input-prompt", - Inputs: map[string]promptkit.ArtifactRef{"input": promptkit.Inline("ignored")}, - }) - if err != nil { - t.Fatalf("prepare with last artifact reader: %v", err) - } - if prepared.Messages[0].Content != "second" { - t.Fatalf("expected last artifact reader, got %#v", prepared.Messages) - } - }) -} - -func TestFallbackProfileSourcePrecedence(t *testing.T) { - const profileID = "application-profile" - - prepareModel := func(t *testing.T, engine *promptkit.Engine, promptID string) string { - t.Helper() - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: promptID}) - if err != nil { - t.Fatalf("prepare: %v", err) - } - return prepared.EffectiveModelParams.Model - } - - t.Run("in-memory profiles override ordinary and fallback profiles", func(t *testing.T) { - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."), - testfixture.WithProfiles(testfixture.Profile{ID: profileID, Endpoint: "http://example.test/v1", Model: "memory-model"}), - testfixture.WithProfileFS(contractProfileFS(profileID, "ordinary-model"), "."), - testfixture.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."), - ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - if model := prepareModel(t, engine, "prompt"); model != "memory-model" { - t.Fatalf("expected in-memory profile, got %q", model) - } - }) - - t.Run("ordinary filesystem source overrides fallback profile", func(t *testing.T) { - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."), - testfixture.WithProfileFS(contractProfileFS(profileID, "ordinary-model"), "."), - testfixture.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."), - ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - if model := prepareModel(t, engine, "prompt"); model != "ordinary-model" { - t.Fatalf("expected ordinary profile, got %q", model) - } - }) - - t.Run("ordinary option replaces configured directory", func(t *testing.T) { - profileDir := t.TempDir() - writePublicProfileFile(t, profileDir, profileID, "http://example.test/v1", "directory-model") - engine, err := testfixture.NewEngine(testfixture.Config{ProfileDir: profileDir}, - testfixture.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."), - testfixture.WithProfileFS(contractProfileFS(profileID, "option-model"), "."), - ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - if model := prepareModel(t, engine, "prompt"); model != "option-model" { - t.Fatalf("expected ordinary option profile, got %q", model) - } - }) - - t.Run("configured directory overrides fallback profile", func(t *testing.T) { - profileDir := t.TempDir() - writePublicProfileFile(t, profileDir, profileID, "http://example.test/v1", "directory-model") - engine, err := testfixture.NewEngine(testfixture.Config{ProfileDir: profileDir}, - testfixture.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."), - testfixture.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."), - ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - if model := prepareModel(t, engine, "prompt"); model != "directory-model" { - t.Fatalf("expected configured directory profile, got %q", model) - } - }) - - t.Run("fallback profile overrides built-in profile", func(t *testing.T) { - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."), - testfixture.WithFallbackProfileFS(contractProfileFS("mistral-small-3", "fallback-model"), "."), - ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - if model := prepareModel(t, engine, "prompt"); model != "fallback-model" { - t.Fatalf("expected fallback profile, got %q", model) - } - }) - - t.Run("missing fallback profile uses built-in profile", func(t *testing.T) { - t.Setenv("OPENROUTER_API_KEY", "test-key") - baseline, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."), - ) - if err != nil { - t.Fatalf("construct baseline engine: %v", err) - } - want := prepareModel(t, baseline, "prompt") - - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."), - testfixture.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."), - ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - if model := prepareModel(t, engine, "prompt"); model != want { - t.Fatalf("expected built-in profile model %q, got %q", want, model) - } - }) -} - -func TestPromptkitOwnedProfilesUseExistingProfileWorkflows(t *testing.T) { - const ( - standardID = "promptkit-openrouter-standard" - autoID = "promptkit-openrouter-auto" - ) - - t.Run("inspection and preparation resolve both names", func(t *testing.T) { - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prompt", standardID, "message"), "."), - ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - - for _, profileID := range []string{standardID, autoID} { - inspection, err := engine.InspectProfile(context.Background(), profileID) - if err != nil { - t.Fatalf("inspect %q: %v", profileID, err) - } - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{ - PromptID: "prompt", - ProfileID: profileID, - APIKey: "test-key", - }) - if err != nil { - t.Fatalf("prepare %q: %v", profileID, err) - } - if inspection.ProfileID != profileID || - inspection.EffectiveModelParams.BackendID != promptkit.BackendOpenRouter || - prepared.SelectedProfileID != profileID || - prepared.SelectedBackendID != promptkit.BackendOpenRouter || - !reflect.DeepEqual(prepared.EffectiveModelParams, inspection.EffectiveModelParams) { - t.Fatalf("inspection and preparation differ for %q: inspection=%#v prepared=%#v", profileID, inspection, prepared) - } - } - }) - - t.Run("consumer definitions override an owned profile and its base", func(t *testing.T) { - auto, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("auto", autoID, "message"), "."), - testfixture.WithProfiles(testfixture.Profile{ - ID: autoID, - Endpoint: "http://consumer.example/v1", - Model: "consumer-auto-model", - }), - ) - if err != nil { - t.Fatalf("construct override engine: %v", err) - } - autoPrepared, err := auto.Prepare(context.Background(), promptkit.RunRequest{PromptID: "auto"}) - if err != nil { - t.Fatalf("prepare overridden profile: %v", err) - } - if autoPrepared.EffectiveModelParams.Model != "consumer-auto-model" { - t.Fatalf("owned profile override model = %q", autoPrepared.EffectiveModelParams.Model) - } - - standard, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("standard", standardID, "message"), "."), - testfixture.WithProfiles(testfixture.Profile{ - ID: "gemini-flash-latest", - Endpoint: "http://consumer.example/v1", - Model: "consumer-base-model", - }), - ) - if err != nil { - t.Fatalf("construct inherited override engine: %v", err) - } - standardPrepared, err := standard.Prepare(context.Background(), promptkit.RunRequest{PromptID: "standard"}) - if err != nil { - t.Fatalf("prepare profile with overridden base: %v", err) - } - if standardPrepared.EffectiveModelParams.Model != "consumer-base-model" { - t.Fatalf("owned profile base override model = %q", standardPrepared.EffectiveModelParams.Model) - } - }) - - t.Run("prompt default selection remains explicit", func(t *testing.T) { - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("default", "application-default", "message"), "."), - testfixture.WithProfiles(testfixture.Profile{ - ID: "application-default", - Endpoint: "http://consumer.example/v1", - Model: "consumer-default-model", - }), - ) - if err != nil { - t.Fatalf("construct default engine: %v", err) - } - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "default"}) - if err != nil { - t.Fatalf("prepare default profile: %v", err) - } - if prepared.SelectedProfileID != "application-default" { - t.Fatalf("selected default profile = %q", prepared.SelectedProfileID) - } - }) -} - -func TestInvalidFallbackProfileSourcesFailConstruction(t *testing.T) { - const profileID = "application-profile" - - t.Run("malformed fallback profile", func(t *testing.T) { - _, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(fstest.MapFS{}, "."), - testfixture.WithFallbackProfileFS(fstest.MapFS{ - "broken.yaml": &fstest.MapFile{Data: []byte("id: broken\nunknown: value\n")}, - }, "."), - ) - if !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("expected invalid configuration, got %v", err) - } - }) - - t.Run("unrelated malformed fallback profile blocks construction", func(t *testing.T) { - _, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."), - testfixture.WithFallbackProfileFS(fstest.MapFS{ - "broken.yaml": &fstest.MapFile{Data: []byte("id: unrelated\nunknown: value\n")}, - "valid.yaml": &fstest.MapFile{Data: []byte("id: application-profile\nendpoint: http://example.test/v1\nmodel: fallback-model\n")}, - }, "."), - ) - if !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("expected invalid configuration, got %v", err) - } - }) - - t.Run("matching malformed fallback profile", func(t *testing.T) { - _, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prompt", "mistral-small-3", "message"), "."), - testfixture.WithFallbackProfileFS(fstest.MapFS{ - "mistral-small-3.yaml": &fstest.MapFile{Data: []byte("id: mistral-small-3\nendpoint: http://example.test/v1\nmodel: fallback-model\nunknown: value\n")}, - }, "."), - ) - if !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("expected invalid configuration, got %v", err) - } - }) - - t.Run("matching malformed ordinary profile", func(t *testing.T) { - _, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."), - testfixture.WithProfileFS(fstest.MapFS{ - "application-profile.yaml": &fstest.MapFile{Data: []byte("id: application-profile\nendpoint: http://example.test/v1\nmodel: ordinary-model\nunknown: value\n")}, - }, "."), - testfixture.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."), - ) - if !errors.Is(err, promptkit.ErrInvalidConfig) { - t.Fatalf("expected invalid configuration, got %v", err) - } - }) -} - -func TestFallbackProfileSourceWorksAcrossWorkflows(t *testing.T) { - const profileID = "application-profile" - client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}} - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prompt", profileID, "message"), "."), - testfixture.WithFallbackProfileFS(contractProfileFS(profileID, "fallback-model"), "."), - promptkit.WithLLMClient(client), - ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - - inspection, err := engine.InspectProfile(context.Background(), profileID) - if err != nil { - t.Fatalf("inspect fallback profile: %v", err) - } - prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"}) - if err != nil { - t.Fatalf("prepare fallback profile: %v", err) - } - preparedExecution, err := engine.PrepareExecution(context.Background(), promptkit.RunRequest{PromptID: "prompt"}) - if err != nil { - t.Fatalf("prepare execution with fallback profile: %v", err) - } - preparedDetails := preparedExecution.Details() - preparedResult, err := engine.RunPrepared(context.Background(), preparedExecution) - if err != nil { - t.Fatalf("run prepared fallback profile: %v", err) - } - runResult, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"}) - if err != nil { - t.Fatalf("run fallback profile: %v", err) - } - - for name, model := range map[string]string{ - "inspection": inspection.EffectiveModelParams.Model, - "preparation": prepared.EffectiveModelParams.Model, - "prepared execution": preparedDetails.EffectiveModelParams.Model, - "prepared result": preparedResult.EffectiveModelParams.Model, - "run result": runResult.EffectiveModelParams.Model, - } { - if model != "fallback-model" { - t.Fatalf("%s model=%q, want fallback-model", name, model) - } - } -} - -func TestEngineSupportsConcurrentPrepareAndRun(t *testing.T) { - engine, err := testfixture.NewEngine(testfixture.Config{}, - testfixture.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), - testfixture.WithBackend(testfixture.Backend{ - ID: "concurrent", - Endpoint: "http://example.test/v1", - }), - testfixture.WithProfiles(testfixture.Profile{ - ID: "profile", - BackendID: "concurrent", - Model: "model", - }), - promptkit.WithLLMClient(countingLLMClient{}), - ) - if err != nil { - t.Fatalf("construct engine: %v", err) - } - - const calls = 40 - errs := make(chan error, calls) - var wg sync.WaitGroup - for i := 0; i < calls; i++ { - wg.Add(1) - go func(run bool) { - defer wg.Done() - request := promptkit.RunRequest{PromptID: "prompt"} - if run { - _, err := engine.Run(context.Background(), request) - errs <- err - return - } - _, err := engine.Prepare(context.Background(), request) - errs <- err - }(i%2 == 0) - } - wg.Wait() - close(errs) - for err := range errs { - if err != nil { - t.Fatalf("concurrent call failed: %v", err) - } - } -} - -type countingLLMClient struct { - calls *atomic.Int64 -} - -func (c countingLLMClient) Generate(context.Context, promptkit.GenerateRequest) (*promptkit.GenerateResponse, error) { - if c.calls != nil { - c.calls.Add(1) - } - return &promptkit.GenerateResponse{Content: "ok"}, nil -} - -type fixedArtifactReader string - -func (r fixedArtifactReader) Read(context.Context, promptkit.ArtifactRef) (*promptkit.Artifact, error) { - return &promptkit.Artifact{Body: []byte(r)}, nil -} - -func contractPromptFS(id, profileID, message string) fstest.MapFS { - return fstest.MapFS{ - "prompt.yaml": &fstest.MapFile{Data: []byte(fmt.Sprintf(`id: %s -version: "1" -default_profile: %s -messages: - - role: user - content: %q -output: - format: text - validation_mode: none -`, id, profileID, message))}, - } -} - -func contractInputPromptFS() fstest.MapFS { - return fstest.MapFS{ - "prompt.yaml": &fstest.MapFile{Data: []byte(`id: input-prompt -version: "1" -default_profile: profile -inputs: - - name: input - required: true -messages: - - role: user - content: '{{input "input"}}' -output: - format: text - validation_mode: none -`)}, - } -} - -func contractProfileFS(id, model string) fstest.MapFS { - return fstest.MapFS{ - "profile.yaml": &fstest.MapFile{Data: []byte(fmt.Sprintf(`id: %s -endpoint: http://example.test/v1 -model: %s -`, id, model))}, - } -} - -func contractSchemaPromptFS() fstest.MapFS { - return fstest.MapFS{ - "prompt.yaml": &fstest.MapFile{Data: []byte(`id: schema-prompt -version: "1" -default_profile: profile -messages: - - role: user - content: message -output: - format: json - validation_mode: json_schema - schema_path: schema.json -`)}, - } -} - -func contractSchemaFS(title string) fstest.MapFS { - return fstest.MapFS{ - "schema.json": &fstest.MapFile{Data: []byte(fmt.Sprintf(`{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "title": %q, - "type": "object" -}`, title))}, - } -} diff --git a/reliable_execution_contract_test.go b/reliable_execution_contract_test.go index 0c82df1..f468667 100644 --- a/reliable_execution_contract_test.go +++ b/reliable_execution_contract_test.go @@ -12,8 +12,8 @@ import ( "testing/fstest" "time" + "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/promptkit" - "gitea.maximumdirect.net/eric/promptkit/testfixture" ) func observedHTTPResponse(status int, body string) *http.Response { @@ -29,7 +29,7 @@ func TestPublicRetryRepairSequenceKeepsUsageAndEventsConsistent(t *testing.T) { } var calls int one := 1 - config := contractConfig(frameworkSchemaDir) + config := simpleConfig("message") config.TransientRetries = &one config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { if calls >= len(responses) { @@ -40,12 +40,12 @@ func TestPublicRetryRepairSequenceKeepsUsageAndEventsConsistent(t *testing.T) { calls++ return r, nil })} - engine, err := testfixture.NewEngine(config) + engine, err := promptkit.NewEngine(config) if err != nil { t.Fatal(err) } var events []promptkit.ExecutionEvent - req := generationErrorRunRequest() + req := simpleGenerationRequest() req.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: 1} req.Observer = func(event promptkit.ExecutionEvent) { events = append(events, event) } result, err := engine.Run(context.Background(), req) @@ -82,6 +82,18 @@ func TestPublicRetryRepairSequenceKeepsUsageAndEventsConsistent(t *testing.T) { } } +func reliabilityConfig(backends, profiles map[string]any) promptkit.Config { + return promptkit.Config{Sources: promptkit.ConfigurationSources{Main: &configkit.Input{Source: configkit.Mapping("reliable execution", map[string]any{ + "backends": backends, + "profiles": profiles, + "prompts": map[string]any{"prompt": map[string]any{"1": map[string]any{ + "default_profile": "profile-a", + "messages": []any{map[string]any{"role": "user", "content": "hello"}}, + "output": map[string]any{"format": "text", "validation_mode": "none"}, + }}}, + })}}} +} + func TestProfilesSharingBackendCoordinateRetryRepairAndAdmission(t *testing.T) { firstA := make(chan struct{}) enteredB := make(chan struct{}) @@ -125,10 +137,14 @@ func TestProfilesSharingBackendCoordinateRetryRepairAndAdmission(t *testing.T) { } }) one := 1 - engine, err := testfixture.NewEngine(testfixture.Config{HTTPClient: &http.Client{Transport: transport}, TransientRetries: &one}, - testfixture.WithPromptFS(contractPromptFS("prompt", "profile-a", "hello"), "."), - testfixture.WithBackend(testfixture.Backend{ID: "limited", Endpoint: "https://example.invalid/v1", ConcurrencyLimit: 1, QueueCapacity: &one}), - testfixture.WithProfiles(testfixture.Profile{ID: "profile-a", BackendID: "limited", Model: "model-a"}, testfixture.Profile{ID: "profile-b", BackendID: "limited", Model: "model-b"})) + config := reliabilityConfig(map[string]any{"limited": map[string]any{ + "endpoint": "https://example.invalid/v1", "concurrency_limit": 1, "queue_capacity": 1, + }}, map[string]any{ + "profile-a": map[string]any{"backend": "limited", "model": "model-a"}, + "profile-b": map[string]any{"backend": "limited", "model": "model-b"}, + }) + config.HTTPClient, config.TransientRetries = &http.Client{Transport: transport}, &one + engine, err := promptkit.NewEngine(config) if err != nil { t.Fatal(err) } @@ -187,11 +203,15 @@ func TestDifferentBackendPoolsCanDispatchTogether(t *testing.T) { return observedHTTPResponse(200, `{"choices":[{"message":{"content":"ok"}}]}`), nil }) zero := 0 - engine, err := testfixture.NewEngine(testfixture.Config{HTTPClient: &http.Client{Transport: transport}, TransientRetries: &zero}, - testfixture.WithPromptFS(contractPromptFS("prompt", "profile-a", "hello"), "."), - testfixture.WithBackend(testfixture.Backend{ID: "backend-a", Endpoint: "https://example.invalid/v1", ConcurrencyLimit: 1, QueueCapacity: &zero}), - testfixture.WithBackend(testfixture.Backend{ID: "backend-b", Endpoint: "https://example.invalid/v1", ConcurrencyLimit: 1, QueueCapacity: &zero}), - testfixture.WithProfiles(testfixture.Profile{ID: "profile-a", BackendID: "backend-a", Model: "model-a"}, testfixture.Profile{ID: "profile-b", BackendID: "backend-b", Model: "model-b"})) + config := reliabilityConfig(map[string]any{ + "backend-a": map[string]any{"endpoint": "https://example.invalid/v1", "concurrency_limit": 1, "queue_capacity": 0}, + "backend-b": map[string]any{"endpoint": "https://example.invalid/v1", "concurrency_limit": 1, "queue_capacity": 0}, + }, map[string]any{ + "profile-a": map[string]any{"backend": "backend-a", "model": "model-a"}, + "profile-b": map[string]any{"backend": "backend-b", "model": "model-b"}, + }) + config.HTTPClient, config.TransientRetries = &http.Client{Transport: transport}, &zero + engine, err := promptkit.NewEngine(config) if err != nil { t.Fatal(err) } @@ -230,7 +250,17 @@ func TestDifferentBackendPoolsCanDispatchTogether(t *testing.T) { func TestPreparedRetriesFreezeSourceAndCredentialAtExecution(t *testing.T) { const envName = "PROMPTKIT_PREPARED_RETRY_KEY" t.Setenv(envName, "first-key") - source := contractPromptFS("prompt", "profile", "original message") + source := fstest.MapFS{"prompt.yaml": &fstest.MapFile{Data: []byte(`prompts: + prompt: + "1": + default_profile: profile + messages: + - role: user + content: original message + output: + format: text + validation_mode: none +`)}} var calls int var auth, content []string transport := roundTripFunc(func(req *http.Request) (*http.Response, error) { @@ -252,8 +282,20 @@ func TestPreparedRetriesFreezeSourceAndCredentialAtExecution(t *testing.T) { return observedHTTPResponse(200, `{"choices":[{"message":{"content":"ok"}}]}`), nil }) one := 1 - engine, err := testfixture.NewEngine(testfixture.Config{HTTPClient: &http.Client{Transport: transport}, TransientRetries: &one}, - testfixture.WithPromptFS(source, "."), testfixture.WithProfiles(testfixture.Profile{ID: "profile", Endpoint: "https://example.invalid/v1", Model: "model"})) + inputs, err := promptkit.FilesystemInputs("prompts", source, ".") + if err != nil { + t.Fatal(err) + } + endpoint, model := "https://example.invalid/v1", "model" + engine, err := promptkit.NewEngine(promptkit.Config{ + Sources: promptkit.ConfigurationSources{Underlays: []configkit.Underlay{ + {Inputs: inputs}, + {Inputs: []configkit.Input{{Source: promptkit.ProfileSource("profile", map[string]promptkit.ProfileConfig{ + "profile": {Endpoint: &endpoint, Model: &model}, + })}}}, + }}, + HTTPClient: &http.Client{Transport: transport}, TransientRetries: &one, + }) if err != nil { t.Fatal(err) } @@ -262,15 +304,16 @@ func TestPreparedRetriesFreezeSourceAndCredentialAtExecution(t *testing.T) { if err != nil { t.Fatal(err) } - source["prompt.yaml"] = &fstest.MapFile{Data: []byte(`id: prompt -version: "1" -default_profile: profile -messages: - - role: user - content: changed message -output: - format: text - validation_mode: none + source["prompt.yaml"] = &fstest.MapFile{Data: []byte(`prompts: + prompt: + "1": + default_profile: profile + messages: + - role: user + content: changed message + output: + format: text + validation_mode: none `)} result, err := engine.RunPrepared(context.Background(), prepared) if err != nil || result == nil || calls != 2 { @@ -290,7 +333,7 @@ func TestPublicNonretryableAndInjectedAttemptsStaySingle(t *testing.T) { three := 3 engine, calls := retryTestEngine(t, &three, func(int) *http.Response { return retryHTTPResponse(400, "") }) var events []promptkit.ExecutionEvent - req := generationErrorRunRequest() + req := simpleGenerationRequest() req.Observer = func(event promptkit.ExecutionEvent) { events = append(events, event) } result, err := engine.Run(context.Background(), req) if result != nil || !errors.Is(err, promptkit.ErrLLMGenerate) || errors.Is(err, promptkit.ErrRetriesExhausted) || *calls != 1 || len(events) != 3 { @@ -314,7 +357,7 @@ func TestPublicExhaustionObservationRetainsProviderCause(t *testing.T) { one := 1 engine, calls := retryTestEngine(t, &one, func(int) *http.Response { return retryHTTPResponse(503, "") }) var events []promptkit.ExecutionEvent - req := generationErrorRunRequest() + req := simpleGenerationRequest() req.Observer = func(event promptkit.ExecutionEvent) { events = append(events, event) } result, err := engine.Run(context.Background(), req) var exhausted *promptkit.RetryExhaustedError @@ -334,7 +377,7 @@ func TestPublicObserverCancellationStopsBeforeRetry(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() var events []promptkit.ExecutionEvent - req := generationErrorRunRequest() + req := simpleGenerationRequest() req.Observer = func(event promptkit.ExecutionEvent) { events = append(events, event) if event.Kind == promptkit.ExecutionAttempt { diff --git a/testdata/framework/fixtures/glossary.yml b/testdata/framework/fixtures/glossary.yml deleted file mode 100644 index 503c677..0000000 --- a/testdata/framework/fixtures/glossary.yml +++ /dev/null @@ -1,2 +0,0 @@ -archive: A catalogued collection of written records. -marker: A small label used to classify an entry. diff --git a/testdata/framework/fixtures/transcript.md b/testdata/framework/fixtures/transcript.md deleted file mode 100644 index b74a773..0000000 --- a/testdata/framework/fixtures/transcript.md +++ /dev/null @@ -1,2 +0,0 @@ -Nia labels the archive. -The archive receives a blue marker. diff --git a/testdata/framework/profiles/contract-fast.yaml b/testdata/framework/profiles/contract-fast.yaml deleted file mode 100644 index 5ac326d..0000000 --- a/testdata/framework/profiles/contract-fast.yaml +++ /dev/null @@ -1,7 +0,0 @@ -id: contract-fast -endpoint: http://localhost:8000/v1 -model: contract-fast-model -temperature: 0.2 -max_tokens: 500 -top_p: 1 -timeout_seconds: 90 diff --git a/testdata/framework/profiles/contract-quality.yaml b/testdata/framework/profiles/contract-quality.yaml deleted file mode 100644 index 4477c74..0000000 --- a/testdata/framework/profiles/contract-quality.yaml +++ /dev/null @@ -1,7 +0,0 @@ -id: contract-quality -endpoint: http://localhost:8000/v1 -model: contract-quality-model -temperature: 0.1 -max_tokens: 1000 -top_p: 0.9 -timeout_seconds: 120 diff --git a/testdata/framework/prompts/contract.markdown_summary.system.md b/testdata/framework/prompts/contract.markdown_summary.system.md deleted file mode 100644 index 3e649a2..0000000 --- a/testdata/framework/prompts/contract.markdown_summary.system.md +++ /dev/null @@ -1 +0,0 @@ -You summarize synthetic archive notes in clear Markdown. diff --git a/testdata/framework/prompts/contract.markdown_summary.user.md b/testdata/framework/prompts/contract.markdown_summary.user.md deleted file mode 100644 index aa47620..0000000 --- a/testdata/framework/prompts/contract.markdown_summary.user.md +++ /dev/null @@ -1,7 +0,0 @@ -Summarize this transcript: - -{{input "transcript"}} - -Optional glossary: - -{{input "glossary"}} diff --git a/testdata/framework/prompts/contract.markdown_summary.yaml b/testdata/framework/prompts/contract.markdown_summary.yaml deleted file mode 100644 index 3242ff5..0000000 --- a/testdata/framework/prompts/contract.markdown_summary.yaml +++ /dev/null @@ -1,20 +0,0 @@ -id: contract.markdown_summary -version: "1.0.0" -default_profile: contract-fast -description: Summarize a synthetic transcript in Markdown. -inputs: - - name: transcript - required: true - content_type: text/markdown - - name: glossary - required: false - content_type: text/yaml -messages: - - role: system - content_file: ./contract.markdown_summary.system.md - - role: user - content_file: ./contract.markdown_summary.user.md -output: - format: markdown - validation_mode: basic - repair_attempts: 0 diff --git a/testdata/framework/prompts/contract.structured_events.system.md b/testdata/framework/prompts/contract.structured_events.system.md deleted file mode 100644 index eba5946..0000000 --- a/testdata/framework/prompts/contract.structured_events.system.md +++ /dev/null @@ -1 +0,0 @@ -Return only JSON that satisfies the requested event schema. diff --git a/testdata/framework/prompts/contract.structured_events.user.md b/testdata/framework/prompts/contract.structured_events.user.md deleted file mode 100644 index 9bcc0cc..0000000 --- a/testdata/framework/prompts/contract.structured_events.user.md +++ /dev/null @@ -1,7 +0,0 @@ -Extract events from this transcript: - -{{input "transcript"}} - -Optional glossary: - -{{input "glossary"}} diff --git a/testdata/framework/prompts/contract.structured_events.yaml b/testdata/framework/prompts/contract.structured_events.yaml deleted file mode 100644 index 607fad8..0000000 --- a/testdata/framework/prompts/contract.structured_events.yaml +++ /dev/null @@ -1,21 +0,0 @@ -id: contract.structured_events -version: "1.0.0" -default_profile: contract-quality -description: Extract synthetic events as structured JSON. -inputs: - - name: transcript - required: true - content_type: text/markdown - - name: glossary - required: false - content_type: text/yaml -messages: - - role: system - content_file: ./contract.structured_events.system.md - - role: user - content_file: ./contract.structured_events.user.md -output: - format: json - validation_mode: json_schema - schema_path: structured_events.schema.json - repair_attempts: 0 diff --git a/testdata/framework/schemas/structured_events.schema.json b/testdata/framework/schemas/structured_events.schema.json deleted file mode 100644 index 4791fe6..0000000 --- a/testdata/framework/schemas/structured_events.schema.json +++ /dev/null @@ -1,19 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "type": "object", - "required": ["events"], - "properties": { - "events": { - "type": "array", - "items": { - "type": "object", - "required": ["title"], - "properties": { - "title": {"type": "string"} - }, - "additionalProperties": false - } - } - }, - "additionalProperties": false -} diff --git a/testfixture/engine.go b/testfixture/engine.go deleted file mode 100644 index 501e9c8..0000000 --- a/testfixture/engine.go +++ /dev/null @@ -1,359 +0,0 @@ -// Package testfixture keeps existing runtime regression scenarios executable -// while their source setup is expressed through the new configuration boundary. -package testfixture - -import ( - "bytes" - "fmt" - "io" - "io/fs" - "net/http" - "os" - "path/filepath" - "strings" - "time" - - "gitea.maximumdirect.net/eric/ConfigKit" - "gitea.maximumdirect.net/eric/promptkit" - "gopkg.in/yaml.v3" -) - -// Config carries the existing regression fixtures' construction settings. -type Config struct { - PromptDir string - ProfileDir string - SchemaDir string - Timeout time.Duration - HTTPClient *http.Client - TransientRetries *int -} - -// Profile describes an existing regression fixture's execution settings. -type Profile struct { - ID, BaseProfileID, BackendID, Endpoint, Model string - Temperature, TopP float64 - MaxTokens, TimeoutSeconds int - ServiceTier, ReasoningEffort string - APIKeyRequired bool - ExtraParams map[string]any -} - -// OpenAICompatibleProfileConfig describes a profile fixture. -type OpenAICompatibleProfileConfig = Profile - -// OpenAICompatibleProfile produces a profile fixture. -func OpenAICompatibleProfile(config OpenAICompatibleProfileConfig) Profile { - profile := Profile(config) - if config.ExtraParams != nil { - profile.ExtraParams = make(map[string]any, len(config.ExtraParams)) - for key, value := range config.ExtraParams { - profile.ExtraParams[key] = value - } - } - return profile -} - -// Backend describes an existing regression fixture's backend settings. -type Backend struct { - ID, Endpoint, APIKeyEnv string - ConcurrencyLimit int - QueueCapacity *int - ExtraParams map[string]any -} - -// LocalBackend returns a conventional local backend fixture. -func LocalBackend(endpoint string, concurrencyLimit int) Backend { - return Backend{ID: promptkit.BackendLocal, Endpoint: endpoint, ConcurrencyLimit: concurrencyLimit} -} - -type option func(*choices) error - -type choices struct { - promptFS, profileFS, fallbackFS fs.FS - promptRoot, profileRoot, fallbackRoot string - promptFile, profileFile string - profiles []Profile - profilesSet bool - backends []Backend -} - -// WithPromptFS selects prompt fixture files. -func WithPromptFS(fsys fs.FS, root string) option { - return func(c *choices) error { - if err := validSourceFS(fsys, root); err != nil { - return err - } - c.promptFS, c.promptRoot, c.promptFile = fsys, root, "" - return nil - } -} - -// WithPromptFile selects one prompt fixture file. -func WithPromptFile(name string) option { - return func(c *choices) error { - if strings.TrimSpace(name) == "" { - return fmt.Errorf("prompt file is required") - } - c.promptFS, c.promptFile = nil, name - return nil - } -} - -// WithProfileFS selects profile fixture files. -func WithProfileFS(fsys fs.FS, root string) option { - return func(c *choices) error { - if err := validSourceFS(fsys, root); err != nil { - return err - } - c.profileFS, c.profileRoot, c.profileFile = fsys, root, "" - return nil - } -} - -// WithProfileFile selects one profile fixture file. -func WithProfileFile(name string) option { - return func(c *choices) error { - if strings.TrimSpace(name) == "" { - return fmt.Errorf("profile file is required") - } - c.profileFS, c.profileFile = nil, name - return nil - } -} - -// WithFallbackProfileFS selects lower-precedence profile fixture files. -func WithFallbackProfileFS(fsys fs.FS, root string) option { - return func(c *choices) error { - if err := validSourceFS(fsys, root); err != nil { - return err - } - c.fallbackFS, c.fallbackRoot = fsys, root - return nil - } -} - -func validSourceFS(filesystem fs.FS, root string) error { - if filesystem == nil || strings.TrimSpace(root) == "" { - return fmt.Errorf("source filesystem and root are required") - } - return nil -} - -// WithProfiles selects in-memory profile fixtures. -func WithProfiles(profiles ...Profile) option { - return func(c *choices) error { c.profiles, c.profilesSet = profiles, true; return nil } -} - -// WithBackend adds one backend fixture. -func WithBackend(backend Backend) option { - if backend.QueueCapacity != nil { - capacity := *backend.QueueCapacity - backend.QueueCapacity = &capacity - } - return func(c *choices) error { c.backends = append(c.backends, backend); return nil } -} - -// NewEngine expresses existing regression scenarios through ConfigKit sources. -func NewEngine(config Config, opts ...any) (*promptkit.Engine, error) { - var choices choices - var runtime []promptkit.Option - for _, candidate := range opts { - switch value := candidate.(type) { - case nil: - case option: - if err := value(&choices); err != nil { - return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err) - } - case promptkit.Option: - runtime = append(runtime, value) - default: - return nil, fmt.Errorf("%w: unsupported fixture option %T", promptkit.ErrInvalidConfig, candidate) - } - } - var underlays []configkit.Underlay - add := func(inputs []configkit.Input, err error) error { - if err != nil { - return err - } - if len(inputs) > 0 { - underlays = append(underlays, configkit.Underlay{Inputs: inputs}) - } - return nil - } - if choices.fallbackFS != nil { - if err := add(sourceInputs("fallback profiles", choices.fallbackFS, choices.fallbackRoot, "profiles")); err != nil { - return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err) - } - } - if choices.profileFile != "" { - if err := add(singleInput(choices.profileFile, "profiles")); err != nil { - return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err) - } - } else if choices.profileFS != nil { - if err := add(sourceInputs("profiles", choices.profileFS, choices.profileRoot, "profiles")); err != nil { - return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err) - } - } else if config.ProfileDir != "" { - if err := add(directoryInputs(config.ProfileDir, "profiles")); err != nil { - return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err) - } - } - if choices.profilesSet { - profiles := make(map[string]promptkit.ProfileConfig, len(choices.profiles)) - for _, value := range choices.profiles { - if _, exists := profiles[value.ID]; exists { - return nil, fmt.Errorf("%w: duplicate profile fixture %q", promptkit.ErrInvalidConfig, value.ID) - } - profiles[value.ID] = profileConfig(value) - } - underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{{Source: promptkit.ProfileSource("memory profiles", profiles)}}}) - } - for index, value := range choices.backends { - config := promptkit.BackendConfig{Endpoint: &value.Endpoint, APIKeyEnv: &value.APIKeyEnv, - ConcurrencyLimit: &value.ConcurrencyLimit, QueueCapacity: value.QueueCapacity, ExtraParams: value.ExtraParams} - underlays = append(underlays, configkit.Underlay{Inputs: []configkit.Input{{Source: promptkit.BackendSource(fmt.Sprintf("backend %d", index), map[string]promptkit.BackendConfig{value.ID: config})}}}) - } - if choices.promptFile != "" { - if err := add(singleInput(choices.promptFile, "prompts")); err != nil { - return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err) - } - } else if choices.promptFS != nil { - if err := add(sourceInputs("prompts", choices.promptFS, choices.promptRoot, "prompts")); err != nil { - return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err) - } - } else if config.PromptDir != "" { - if err := add(directoryInputs(config.PromptDir, "prompts")); err != nil { - return nil, fmt.Errorf("%w: %w", promptkit.ErrInvalidConfig, err) - } - } - return promptkit.NewEngine(promptkit.Config{Sources: promptkit.ConfigurationSources{Underlays: underlays}, - SchemaDir: config.SchemaDir, Timeout: config.Timeout, HTTPClient: config.HTTPClient, - TransientRetries: config.TransientRetries}, runtime...) -} - -func profileConfig(value Profile) promptkit.ProfileConfig { - config := promptkit.ProfileConfig{ExtraParams: value.ExtraParams} - if value.BaseProfileID != "" { - config.Extends = &value.BaseProfileID - } - if value.BackendID != "" { - config.Backend = &value.BackendID - } - if value.Endpoint != "" { - config.Endpoint = &value.Endpoint - } - if value.Model != "" { - config.Model = &value.Model - } - if value.Temperature != 0 { - config.Temperature = &value.Temperature - } - if value.TopP != 0 { - config.TopP = &value.TopP - } - if value.MaxTokens != 0 { - config.MaxTokens = &value.MaxTokens - } - if value.TimeoutSeconds != 0 { - config.TimeoutSeconds = &value.TimeoutSeconds - } - if value.ServiceTier != "" { - config.ServiceTier = &value.ServiceTier - } - if value.ReasoningEffort != "" { - config.ReasoningEffort = &value.ReasoningEffort - } - if value.APIKeyRequired { - config.APIKeyRequired = &value.APIKeyRequired - } - return config -} - -func directoryInputs(directory, collection string) ([]configkit.Input, error) { - if _, err := os.Stat(directory); err != nil { - return nil, err - } - return sourceInputs(directory, os.DirFS(directory), ".", collection) -} - -func sourceInputs(label string, filesystem fs.FS, root, collection string) ([]configkit.Input, error) { - return promptkit.FilesystemInputs(label, convertingFS{FS: filesystem, collection: collection}, root) -} - -func singleInput(name, collection string) ([]configkit.Input, error) { - info, err := os.Stat(name) - if err != nil { - return nil, err - } - if info.IsDir() { - return nil, fmt.Errorf("source %s is a directory", name) - } - filesystem := convertingFS{FS: os.DirFS(filepath.Dir(name)), collection: collection} - namespace := configkit.NewFilesystem(name, filesystem) - return []configkit.Input{{Source: namespace.File(filepath.Base(name)), Filesystem: &configkit.FilesystemContext{FS: &namespace, Root: "."}}}, nil -} - -type convertingFS struct { - fs.FS - collection string -} - -func (source convertingFS) Open(name string) (fs.File, error) { - file, err := source.FS.Open(name) - if err != nil || (!strings.HasSuffix(strings.TrimSpace(name), ".yaml") && !strings.HasSuffix(strings.TrimSpace(name), ".yml")) { - return file, err - } - info, err := file.Stat() - if err != nil { - file.Close() - return nil, err - } - data, err := io.ReadAll(file) - closeErr := file.Close() - if err != nil { - return nil, err - } - if closeErr != nil { - return nil, closeErr - } - converted := convertDocument(data, source.collection) - return &memoryFile{Reader: bytes.NewReader(converted), info: info}, nil -} - -type memoryFile struct { - *bytes.Reader - info fs.FileInfo -} - -func (file *memoryFile) Stat() (fs.FileInfo, error) { return file.info, nil } -func (file *memoryFile) Close() error { return nil } - -func convertDocument(data []byte, collection string) []byte { - var document map[string]any - if err := yaml.Unmarshal(data, &document); err != nil || document == nil { - return data - } - if _, keyed := document[collection]; keyed { - return data - } - id, idOK := document["id"] - if !idOK { - return data - } - delete(document, "id") - if collection == "profiles" { - if base, exists := document["base_profile"]; exists { - delete(document, "base_profile") - document["extends"] = base - } - data, _ = yaml.Marshal(map[string]any{"profiles": map[string]any{fmt.Sprint(id): document}}) - return data - } - version, versionOK := document["version"] - if !versionOK { - return data - } - delete(document, "version") - data, _ = yaml.Marshal(map[string]any{"prompts": map[string]any{fmt.Sprint(id): map[string]any{fmt.Sprint(version): document}}}) - return data -} diff --git a/transient_retries_contract_test.go b/transient_retries_contract_test.go index 490e62d..a628582 100644 --- a/transient_retries_contract_test.go +++ b/transient_retries_contract_test.go @@ -11,7 +11,6 @@ import ( "testing" "gitea.maximumdirect.net/eric/promptkit" - "gitea.maximumdirect.net/eric/promptkit/testfixture" ) func retryHTTPResponse(status int, content string) *http.Response { @@ -25,13 +24,13 @@ func retryHTTPResponse(status int, content string) *http.Response { func retryTestEngine(t *testing.T, budget *int, next func(int) *http.Response) (*promptkit.Engine, *int) { t.Helper() calls := 0 - config := contractConfig(frameworkSchemaDir) + config := simpleConfig("message") config.TransientRetries = budget config.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { calls++ return next(calls), nil })} - engine, err := testfixture.NewEngine(config) + engine, err := promptkit.NewEngine(config) if err != nil { t.Fatal(err) } @@ -45,7 +44,7 @@ func TestBuiltInRetryDefaultsAndRequestOverride(t *testing.T) { } return retryHTTPResponse(200, "ok") }) - result, err := engine.Run(context.Background(), generationErrorRunRequest()) + result, err := engine.Run(context.Background(), simpleGenerationRequest()) if err != nil || result == nil || result.RawOutput != "ok" || *calls != 3 { t.Fatalf("default result=%+v, error=%v, calls=%d", result, err, *calls) } @@ -54,7 +53,7 @@ func TestBuiltInRetryDefaultsAndRequestOverride(t *testing.T) { engine, calls = retryTestEngine(t, &engineBudget, func(int) *http.Response { return retryHTTPResponse(503, "") }) engineBudget = 5 // Construction copied the configured value. zero := 0 - req := generationErrorRunRequest() + req := simpleGenerationRequest() req.TransientRetries = &zero _, err = engine.Run(context.Background(), req) if !errors.Is(err, promptkit.ErrLLMGenerate) || errors.Is(err, promptkit.ErrRetriesExhausted) || *calls != 1 { @@ -70,14 +69,14 @@ func TestBuiltInRetryDefaultsAndRequestOverride(t *testing.T) { func TestRetryValidationAndPreparedFreezing(t *testing.T) { negative := -1 - config := contractConfig(frameworkSchemaDir) + config := simpleConfig("message") config.TransientRetries = &negative - if _, err := testfixture.NewEngine(config); !errors.Is(err, promptkit.ErrInvalidConfig) { + if _, err := promptkit.NewEngine(config); !errors.Is(err, promptkit.ErrInvalidConfig) { t.Fatalf("negative engine budget = %v", err) } engine, calls := retryTestEngine(t, nil, func(int) *http.Response { return retryHTTPResponse(503, "") }) - req := generationErrorRunRequest() + req := simpleGenerationRequest() req.TransientRetries = &negative if _, err := engine.Prepare(context.Background(), req); !errors.Is(err, promptkit.ErrInvalidRequest) { t.Fatalf("Prepare negative budget = %v", err) @@ -95,7 +94,7 @@ func TestRetryValidationAndPreparedFreezing(t *testing.T) { largeBudgetEngine, largeCalls := retryTestEngine(t, &maximum, func(int) *http.Response { return retryHTTPResponse(200, "ok") }) - if _, err := largeBudgetEngine.Run(context.Background(), generationErrorRunRequest()); err != nil || *largeCalls != 1 { + if _, err := largeBudgetEngine.Run(context.Background(), simpleGenerationRequest()); err != nil || *largeCalls != 1 { t.Fatalf("large valid budget: error=%v, calls=%d", err, *largeCalls) } @@ -124,7 +123,7 @@ func TestBuiltInRetryBudgetAppliesToRepairGeneration(t *testing.T) { return retryHTTPResponse(200, `{"ok":true}`) } }) - req := generationErrorRunRequest() + req := simpleGenerationRequest() req.Validation = &promptkit.OutputContract{Format: promptkit.FormatJSON, ValidationMode: promptkit.ValidationJSON, RepairAttempts: 1} result, err := engine.Run(context.Background(), req) if err != nil || result == nil || !result.Validation.IsValid || *calls != 4 { @@ -135,7 +134,7 @@ func TestBuiltInRetryBudgetAppliesToRepairGeneration(t *testing.T) { func TestRetryExhaustionRetainsPublicProviderCause(t *testing.T) { one := 1 engine, calls := retryTestEngine(t, &one, func(int) *http.Response { return retryHTTPResponse(503, "") }) - result, err := engine.Run(context.Background(), generationErrorRunRequest()) + result, err := engine.Run(context.Background(), simpleGenerationRequest()) if result != nil || *calls != 2 || !errors.Is(err, promptkit.ErrRetriesExhausted) || !errors.Is(err, promptkit.ErrLLMGenerate) { t.Fatalf("exhaustion result=%+v, error=%v, calls=%d", result, err, *calls) } @@ -161,9 +160,9 @@ func TestRetryExhaustionRetainsPublicProviderCause(t *testing.T) { func TestInjectedClientIsNotAutomaticallyRetried(t *testing.T) { client := &fakeLLMClient{err: errors.New("injected failure")} - engine := newContractEngineWithOptions(t, frameworkSchemaDir, promptkit.WithLLMClient(client)) + engine := newSimpleEngine(t, "message", promptkit.WithLLMClient(client)) count := 3 - req := generationErrorRunRequest() + req := simpleGenerationRequest() req.TransientRetries = &count _, err := engine.Run(context.Background(), req) if len(client.requests) != 1 || !errors.Is(err, client.err) || errors.Is(err, promptkit.ErrRetriesExhausted) {