From 359b7313f4e6a1a703769b6d44333c6178f77112 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Wed, 29 Jul 2026 17:22:55 +0000 Subject: [PATCH] Harden backend contracts and documentation --- backends.go | 3 +- doc.go | 21 ++-- docs/consumers/pkg-promptkit.md | 14 ++- docs/formats.md | 9 +- docs/integrations/openai-compatible-chat.md | 30 +++-- docs/internal/llm.md | 11 +- docs/internal/overview.md | 12 +- docs/internal/runner.md | 13 ++- docs/internal/sources.md | 3 +- docs/policy/architecture.md | 34 +++--- docs/roadmap/backends.md | 2 + docs/roadmap/implementation.md | 2 + engine.go | 18 +-- engine_test.go | 86 ++++++++++++++ public_contract_test.go | 123 +++++++++++++++----- types.go | 47 ++++---- 16 files changed, 315 insertions(+), 113 deletions(-) diff --git a/backends.go b/backends.go index 5d3cb1b..0ecc039 100644 --- a/backends.go +++ b/backends.go @@ -5,7 +5,8 @@ import ( "gitea.maximumdirect.net/eric/promptkit/internal/domain" ) -// BackendOpenRouter identifies Promptkit's built-in OpenRouter backend. +// BackendOpenRouter is the reserved ID of Promptkit's built-in OpenRouter +// backend. const BackendOpenRouter = backend.OpenRouterID // Backend configures one engine-scoped OpenAI-compatible backend. diff --git a/doc.go b/doc.go index 5cd6732..c26e2b2 100644 --- a/doc.go +++ b/doc.go @@ -2,9 +2,10 @@ // prompt-defined LLM workflows. // // Applications construct an [Engine] with [NewEngine], select filesystem or -// in-memory sources with options, and call [Engine.Prepare] or [Engine.Run]. -// Concrete repositories, validators, and the built-in OpenAI-compatible client -// remain internal implementation details. +// in-memory sources and optional engine-scoped [Backend] registrations, and +// call [Engine.Prepare] or [Engine.Run]. Concrete registries, repositories, +// validators, and the built-in OpenAI-compatible client remain internal +// implementation details. // // # Concurrency and ownership // @@ -12,11 +13,11 @@ // or [ArtifactReader] can therefore receive concurrent calls and must be safe // for that use. // -// NewEngine copies in-memory profiles. Prepare and Run copy request maps, -// slices, pointer values, and JSON-compatible extra parameters before using -// them. Returned values and values passed to extension interfaces are likewise -// isolated from engine state. Callers own those copies and may mutate them -// after the call that supplied or returned them. +// NewEngine copies in-memory profiles and backend definitions. Prepare and Run +// copy request maps, slices, pointer values, and JSON-compatible extra +// parameters before using them. Returned values and values passed to extension +// interfaces are likewise isolated from engine state. Callers own those copies +// and may mutate them after the call that supplied or returned them. // // # Security and sensitive data // @@ -42,8 +43,8 @@ // [GenerateResponse], [ExecutionTargetPresence], and the string value types // used by those values. // -// Construction values, including [Config], [RunRequest], [ArtifactRef], -// [ExecutionTargetOverride], [Profile], and +// Construction values, including [Config], [Backend], [RunRequest], +// [ArtifactRef], [ExecutionTargetOverride], [Profile], and // [OpenAICompatibleProfileConfig], do not have stable JSON representations. // Direct API keys are nevertheless excluded from JSON for every public value. // diff --git a/docs/consumers/pkg-promptkit.md b/docs/consumers/pkg-promptkit.md index aec262e..76e343d 100644 --- a/docs/consumers/pkg-promptkit.md +++ b/docs/consumers/pkg-promptkit.md @@ -33,8 +33,8 @@ engine, err := promptkit.NewEngine(promptkit.Config{ }) ``` -Options support single-file or `fs.FS` sources, in-memory profiles, and -injected artifact or model clients. Consult the +Options support single-file or `fs.FS` sources, in-memory profiles, +engine-scoped backends, and injected artifact or model clients. Consult the [constructor and option GoDoc](../../engine.go) for composition, precedence, validation, and default transport behavior. Source discovery, format validation, and profile precedence are defined by the @@ -123,6 +123,13 @@ Registrations belong to one engine and custom IDs cannot replace built-ins. The [`Backend` and `WithBackend` GoDoc](../../backends.go) defines validation, copying, uniqueness, and request-default behavior. +Both file-backed and in-memory profiles select a registration through +`backend` or `Profile.BackendID`. Profile and request endpoint overrides retain +that routing identity. `PreparedRun.SelectedBackendID`, +`RunResult.SelectedBackendID`, and the effective `ExecutionTarget.BackendID` +expose it to consumers and injected model clients. Endpoint-only profiles +remain supported and expose an empty backend ID. + ## Credentials File-backed profiles name an environment variable; in-memory profiles can @@ -163,7 +170,8 @@ distinguish invalid construction, invalid requests, absent sources, source-loading failures, collaborator failures, and operational validation failures. Specific request conditions may also match the broader `ErrInvalidRequest`, and injected collaborator identities are preserved where -documented. +documented. Invalid or duplicate backend registrations match +`ErrInvalidConfig`; selecting an unknown backend matches `ErrProfileLoad`. ## Application Boundary diff --git a/docs/formats.md b/docs/formats.md index 1e4ce2a..b7aaeca 100644 --- a/docs/formats.md +++ b/docs/formats.md @@ -168,6 +168,10 @@ variable name in `api_key_env`. Promptkit does not infer a backend from a model or endpoint. Endpoint-only profiles remain supported and have no effective backend ID. +The engine always provides the built-in `openrouter` ID. Consumers can add +engine-scoped IDs with +[`WithBackend`](../backends.go); exact registration validation belongs to its +GoDoc. `extra_params` accepts null, booleans, finite numbers, strings, arrays, and objects with string keys. Keys must be non-empty. With the built-in client, @@ -226,8 +230,9 @@ invalid matching profile is an error and does not fall back. In-memory Every built-in selects the `openrouter` backend. The engine's built-in backend registry supplies `https://openrouter.ai/api/v1` and the environment-variable name `OPENROUTER_API_KEY`, so individual profiles contain only model and -generation settings. A custom or in-memory profile with the same profile ID -takes precedence. +generation settings. Built-in profile files do not repeat those connection +values. A custom or in-memory profile with the same profile ID takes +precedence. | Provider | ID | Model | | --- | --- | --- | diff --git a/docs/integrations/openai-compatible-chat.md b/docs/integrations/openai-compatible-chat.md index 92f50d4..7c064e4 100644 --- a/docs/integrations/openai-compatible-chat.md +++ b/docs/integrations/openai-compatible-chat.md @@ -13,10 +13,15 @@ that produce these outbound settings. ## Endpoint And Method Generation sends an HTTP `POST` with `Content-Type: application/json`. -A non-empty endpoint from the execution target overrides the client's -configured base URL. After trailing slashes are removed, -`/chat/completions` is appended. Generation fails before sending when neither -source supplies an endpoint. +Before the client is called, the engine resolves framework, backend, profile, +and request values into one execution target. A non-empty endpoint from that +target overrides the client's configured base URL. After trailing slashes are +removed, `/chat/completions` is appended. Generation fails before sending when +neither source supplies an endpoint. + +The target's backend ID is routing metadata for prepared values, results, and +injected clients. The built-in client does not derive the URL from that ID and +does not serialize it in the provider request. ## Authentication @@ -26,6 +31,12 @@ the client reads that variable and requires a non-empty value. The selected key is sent as `Authorization: Bearer `. No authorization header is sent when neither mechanism is configured. +The target contains the already resolved environment-variable name: an +explicit request override takes precedence over profile metadata, which takes +precedence over the backend default. Only the name reaches prepared metadata; +the environment value is read just before the provider call and is never added +to the JSON body. + ## Request Body The request body always contains `model` and `messages`. The execution @@ -47,9 +58,11 @@ The client conditionally includes: - `response_format` for JSON Schema structured output, including its name, strict flag, and schema document. -Extra parameters are merged directly into the top-level body after JSON -serialization is verified. Empty keys and collisions with these reserved -fields are rejected before any provider call: +The engine resolves backend, profile, and request extra-parameter maps by +whole-map replacement rather than key merging. The resulting effective map is +then merged directly into the top-level body after JSON serialization is +verified. Empty keys and collisions with these reserved fields are rejected +before any provider call: - `model` - `session_id` @@ -61,6 +74,9 @@ fields are rejected before any provider call: - `reasoning_effort` - `response_format` +`backend_id`, `api_key_env`, and resolved credential values are not provider +request fields. + ## Response Handling Any 2xx response is decoded as an OpenAI-compatible chat response. The client diff --git a/docs/internal/llm.md b/docs/internal/llm.md index fa9f72d..e441508 100644 --- a/docs/internal/llm.md +++ b/docs/internal/llm.md @@ -20,6 +20,11 @@ orchestration. `OpenAICompatibleClient` is the built-in implementation. It uses internal domain values for rendered prompts, execution targets, structured output, responses, and token usage. +The runner supplies a fully resolved target after applying backend, profile, +and request precedence. The client uses its endpoint, credential metadata, +generation fields, and extra parameters. `BackendID` remains routing metadata +for the generation boundary and is not mapped into the provider payload. + Construction validates the configured base URL and clones any supplied `http.Client` so Promptkit can apply its timeout default without mutating the caller's client. Generation then: @@ -51,5 +56,7 @@ The [OpenAI-compatible client tests](../../internal/llm/openai_compatible_client_test.go) own configuration, client cloning, deterministic deadline precedence, authentication, request and response mapping, malformed data, error identity, -cancellation, and response-body suppression. They use local test servers and -test transports; the default suite makes no live or paid provider requests. +cancellation, and response-body suppression. The root transport contract test +also verifies that resolved backend settings reach this client without +serializing backend identity. All use local test servers or test transports; +the default suite makes no live or paid provider requests. diff --git a/docs/internal/overview.md b/docs/internal/overview.md index d080866..9df10b2 100644 --- a/docs/internal/overview.md +++ b/docs/internal/overview.md @@ -11,21 +11,21 @@ contributor workflow and validation. | Component | Implemented responsibility | References | | --- | --- | --- | -| Root `promptkit` package | Provides the supported engine facade, source and injection options, public request and result values, built-in profile construction, extension interfaces, value conversion, redacted formatting, and public error mapping. | [Package GoDoc](../../doc.go), [engine assembly](../../engine.go) | +| Root `promptkit` package | Provides the supported engine facade, source, backend-registration, and injection options, public request and result values, profile construction, extension interfaces, value conversion, redacted formatting, and public error mapping. | [Package GoDoc](../../doc.go), [backend API](../../backends.go), [engine assembly](../../engine.go) | | `examples/go-library/prepare` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, and `Prepare`. It is not a public library package. | [Example program](../../examples/go-library/prepare/main.go) | | `examples/go-library/run` | Demonstrates an offline downstream consumer using a prompt file, in-memory profile, inline input, an injected deterministic model client, and `Run`. It is not a public library package. | [Example program](../../examples/go-library/run/main.go) | -| `internal/backend` | Validates and defensively copies immutable OpenAI-compatible backend definitions, supplies the built-in OpenRouter definition, and owns the shared reserved request-field rule. | [Backend registry](../../internal/backend/registry.go) | +| `internal/backend` | Constructs each engine's immutable registry from the built-in OpenRouter definition and consumer additions, validates and defensively copies definitions, and owns the shared reserved request-field rule. | [Backend registry](../../internal/backend/registry.go) | | `internal/domain` | Defines internal framework values for requests, artifacts, prompt definitions, profiles, execution targets, rendering, generation, and validation. | [Domain declarations](../../internal/domain/domain.go) | | `internal/defaults` | Defines application-neutral framework constants and constructs the default execution target. It contains no CLI, server, or inbound HTTP limits. | [Framework defaults](../../internal/defaults/defaults.go) | | `internal/filecatalog` | Provides deterministic YAML discovery and path helpers for operating-system filesystems and `fs.FS` sources. | [File catalog](../../internal/filecatalog/catalog.go) | | `internal/promptdef` | Loads strictly decoded, validated prompt definitions from filesystem and `fs.FS` sources, including version selection and contained file-backed message content. | [Framework formats](../formats.md), [prompt-definition repository](../../internal/promptdef/filesystem_repository.go) | -| `internal/profile` | Loads strictly decoded, validated execution profiles from filesystem and `fs.FS` sources and composes repositories with error-preserving fallback. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go) | -| `internal/profile/builtin` | Embeds the built-in execution profile catalog and combines it with an optional primary repository. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) | +| `internal/profile` | Loads strictly decoded, validated execution profiles, including backend selection, from filesystem and `fs.FS` sources and composes repositories with error-preserving fallback. | [Framework formats](../formats.md), [profile repositories](../../internal/profile/filesystem_repository.go) | +| `internal/profile/builtin` | Embeds the built-in profile catalog, whose entries select OpenRouter, and combines it with an optional primary repository. | [Built-in catalog](../formats.md#built-in-profile-catalog), [repository](../../internal/profile/builtin/repository.go) | | `internal/prompt` | Renders prompt messages from Go templates with artifact, variable, session, and cache-control data. | [Go-template renderer](../../internal/prompt/go_renderer.go) | | `internal/artifact` | Resolves ordinary inline and unrestricted caller-selected file references into copied artifacts with metadata and hashes. | [Internal sources and validation](sources.md) | | `internal/validate` | Validates basic, JSON, and JSON Schema output using operating-system filesystem or `fs.FS` schema sources. | [Framework formats](../formats.md#schemas), [internal sources and validation](sources.md) | -| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests, response decoding, authentication, and deadline handling. | [Internal model client](llm.md) | -| `internal/usecase` | Coordinates preparation and execution across internal sources, rendering, artifact loading, generation, validation, and optional repair. | [Internal runner](runner.md) | +| `internal/llm` | Defines the internal generation boundary and implements outbound OpenAI-compatible chat requests from resolved execution targets, including response decoding, authentication, and deadline handling. | [Internal model client](llm.md) | +| `internal/usecase` | Resolves backend, profile, and request settings and coordinates preparation and execution across internal sources, rendering, artifact loading, generation, validation, and optional repair. | [Internal runner](runner.md) | The root package assembles these internal components without exposing their representations. Consumers depend only on the root facade. diff --git a/docs/internal/runner.md b/docs/internal/runner.md index 0b51110..2831a21 100644 --- a/docs/internal/runner.md +++ b/docs/internal/runner.md @@ -18,10 +18,11 @@ and override semantics consumed by the runner. `Runner` coordinates narrow internal interfaces for prompt definitions, profiles, backend resolution, artifacts, rendering, model generation, and -validation. The root engine supplies an immutable built-in backend registry. -Schema documents are loaded through the validator's optional schema-loader -interface. An output repairer can be injected internally, but the ordinary -runner constructor does not enable one. +validation. The root engine supplies one immutable registry containing the +built-in backend and validated consumer additions. Schema documents are loaded +through the validator's optional schema-loader interface. An output repairer +can be injected internally, but the ordinary runner constructor does not +enable one. Each invocation carries its state in request, prepared-run, and result values. The runner has no durable run or session store. @@ -53,6 +54,10 @@ apply in that order. A profile requiring a direct key clears an inherited backend environment name unless the request supplies its own name. Secret values remain excluded from serialized metadata. +The registry is read-only after engine construction. Concurrent `Prepare` and +`Run` calls resolve independent defensive backend values and keep all +invocation state local. + ## Run Flow `Run` calls `Prepare` rather than maintaining a second preparation path. It diff --git a/docs/internal/sources.md b/docs/internal/sources.md index 33d22db..ad241c0 100644 --- a/docs/internal/sources.md +++ b/docs/internal/sources.md @@ -36,7 +36,8 @@ profile selects `openrouter` and inherits its endpoint and credential environment-variable name from the built-in backend registry rather than repeating those values. Profile behavior is owned by the [profile repository tests](../../internal/profile/repository_test.go), while -catalog completeness, backend-selection invariant, duplicate IDs, and overlay behavior are owned by the +catalog completeness, the backend-selection invariant, duplicate IDs, and +overlay behavior are owned by the [built-in repository tests](../../internal/profile/builtin/repository_test.go). ## Ordinary Artifacts diff --git a/docs/policy/architecture.md b/docs/policy/architecture.md index 4ba342c..c114392 100644 --- a/docs/policy/architecture.md +++ b/docs/policy/architecture.md @@ -47,17 +47,19 @@ The `examples/go-library/prepare` and `examples/go-library/run` packages are maintained downstream consumers of the root facade. They do not expose library packages or participate in internal assembly. -The root facade assembles the internal repositories, renderer, validator, -outbound client, and use-case runner while translating public values and -errors at the library boundary. The defaults and renderer depend on the domain -model. Prompt-definition and profile repositories use the domain model, file -catalog, and YAML decoder. The built-in profile repository supplies an -embedded `fs.FS` to the profile package. Artifact reading uses the domain model -and application-neutral defaults. Validation uses the domain model, file -catalog, and JSON Schema implementation. The model client uses the domain -model, application-neutral defaults, and an injected or standard-library HTTP -client. The use-case runner depends on the narrow interfaces owned by each -internal component. +The root facade assembles one immutable backend registry, the internal +repositories, renderer, validator, outbound client, and use-case runner while +translating public values and errors at the library boundary. The registry +contains built-ins plus validated engine-scoped consumer additions. The +defaults and renderer depend on the domain model. Prompt-definition and +profile repositories use the domain model, file catalog, and YAML decoder. The +built-in profile repository supplies an embedded `fs.FS` to the profile +package. Artifact reading uses the domain model and application-neutral +defaults. Validation uses the domain model, file catalog, and JSON Schema +implementation. The model client uses the domain model, application-neutral +defaults, and an injected or standard-library HTTP client. The use-case runner +depends on the narrow interfaces owned by each internal component, including +backend lookup. The current implementation follows this dependency direction: @@ -74,11 +76,11 @@ downstream consumers, including Scriptorium narrow injected abstractions ``` -The backend registry depends on the domain model, and the model client reuses -its OpenAI-compatible reserved request-field rule. The facade coordinates -internal components and adapts the supported public extension interfaces to -narrow internal abstractions. Internal components must not depend on consumers -or on Scriptorium. +The backend registry depends on the domain model, has no mutation API after +construction, and shares its OpenAI-compatible reserved request-field rule +with the model client. The facade coordinates internal components and adapts +the supported public extension interfaces to narrow internal abstractions. +Internal components must not depend on consumers or on Scriptorium. ## Repository And Consumer Boundary diff --git a/docs/roadmap/backends.md b/docs/roadmap/backends.md index d2a7816..8116c00 100644 --- a/docs/roadmap/backends.md +++ b/docs/roadmap/backends.md @@ -1,5 +1,7 @@ # Extensible LLM Backend Registry +**Status:** Complete. + ## Purpose This roadmap defines the scope and target end state for formal diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index aacb68b..70e5e0f 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -428,6 +428,8 @@ external endpoint. Keep lower-level registry validation cases in ## Stage 4 — Contract Hardening And Documentation Completion +**Status:** Complete. + ### Goal Audit the completed feature across public, profile, use-case, and transport diff --git a/engine.go b/engine.go index b1eb53b..4f6148c 100644 --- a/engine.go +++ b/engine.go @@ -25,7 +25,8 @@ import ( ) // ErrInvalidConfig identifies invalid engine construction, including missing -// required configuration, invalid options, and a nil Engine receiver. +// required configuration, invalid options or backend registrations, and a nil +// Engine receiver. var ErrInvalidConfig = errors.New("invalid engine configuration") var ( @@ -48,8 +49,8 @@ var ( // ErrPromptNotFound. ErrPromptLoad = errors.New("failed to load prompt definition") // ErrProfileLoad identifies a failure to read, decode, validate, or select - // an execution profile, except for the not-found case represented by - // ErrProfileNotFound. + // an execution profile or resolve its backend, except for the profile + // not-found case represented by ErrProfileNotFound. ErrProfileLoad = errors.New("failed to load execution profile") // ErrAPIKeyEnvMissing identifies an APIKeyEnv whose environment variable is // unset or empty when no direct RunRequest.APIKey takes precedence. Such an @@ -407,11 +408,12 @@ func fileSource(name string) (fs.FS, string, error) { // Prepare resolves and renders a prompt request without calling an LLM. // -// Prepare selects the prompt and profile, resolves effective execution -// settings and the output contract, loads and hashes inputs, loads structured -// output schema metadata when required, and renders the session ID and -// messages. The returned PreparedRun is owned by the caller and never contains -// a resolved API-key value, model output, or validation result. +// Prepare selects the prompt and profile, resolves any selected backend and +// effective execution settings, resolves the output contract, loads and hashes +// inputs, loads structured-output schema metadata when required, and renders +// the session ID and messages. The returned PreparedRun is owned by the caller +// and never contains a resolved API-key value, model output, or validation +// result. // // A nil Engine returns an error matching ErrInvalidConfig. Request and // preparation failures may match ErrInvalidRequest, ErrPromptNotFound, diff --git a/engine_test.go b/engine_test.go index 93d2c88..00753d0 100644 --- a/engine_test.go +++ b/engine_test.go @@ -755,6 +755,92 @@ func TestRunUsesDirectAPIKeyWithDefaultLLMClient(t *testing.T) { } } +func TestRunUsesResolvedBackendWithBuiltInLLMClient(t *testing.T) { + const ( + backendID = "local-test" + envName = "PROMPTKIT_BACKEND_TRANSPORT_KEY" + apiKey = "synthetic-backend-key" + ) + t.Setenv(envName, apiKey) + + var ( + gotAuth string + gotBody map[string]any + ) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotAuth = r.Header.Get("Authorization") + if r.URL.Path != "/v1/chat/completions" { + t.Errorf("unexpected path: %s", r.URL.Path) + } + if err := json.NewDecoder(r.Body).Decode(&gotBody); err != nil { + t.Errorf("decode request body: %v", err) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{ + "choices": [{"message": {"role": "assistant", "content": "# Summary\n\nDone."}}], + "usage": {"prompt_tokens": 3, "completion_tokens": 4, "total_tokens": 7} +}`)) + })) + defer server.Close() + + engine, err := promptkit.NewEngine(promptkit.Config{ + PromptDir: frameworkPromptDir, + SchemaDir: frameworkSchemaDir, + }, + promptkit.WithBackend(promptkit.Backend{ + ID: backendID, + Endpoint: server.URL + "/v1", + APIKeyEnv: envName, + ExtraParams: map[string]any{ + "provider": "synthetic", + }, + }), + promptkit.WithProfiles(promptkit.Profile{ + ID: "backend-transport", + BackendID: backendID, + Model: "test-model", + }), + ) + if err != nil { + t.Fatalf("construct engine: %v", err) + } + + result, err := engine.Run(context.Background(), promptkit.RunRequest{ + PromptID: frameworkMarkdownSummaryPromptID, + ProfileID: "backend-transport", + Inputs: map[string]promptkit.ArtifactRef{ + "transcript": promptkit.Inline("Rin opens the gate."), + "glossary": promptkit.Inline("gate: A guarded passage."), + }, + }) + if err != nil { + t.Fatalf("run with resolved backend: %v", err) + } + if gotAuth != "Bearer "+apiKey { + t.Fatalf("unexpected Authorization header: %q", gotAuth) + } + if gotBody["model"] != "test-model" || gotBody["provider"] != "synthetic" { + t.Fatalf("backend defaults did not reach provider payload: %#v", gotBody) + } + for _, field := range []string{"backend_id", "api_key_env"} { + if _, ok := gotBody[field]; ok { + t.Fatalf("internal metadata field %q was serialized to provider payload: %#v", field, gotBody) + } + } + bodyJSON, err := json.Marshal(gotBody) + if err != nil { + t.Fatalf("marshal captured provider payload: %v", err) + } + if strings.Contains(string(bodyJSON), apiKey) { + t.Fatalf("credential value was serialized to provider payload: %s", bodyJSON) + } + if result.SelectedBackendID != backendID || + result.EffectiveModelParams.Endpoint != server.URL+"/v1" || + result.EffectiveModelParams.APIKeyEnv != envName { + t.Fatalf("unexpected resolved backend metadata: %+v", result) + } +} + func TestPrepareDirectAPIKeyBypassesMissingEnvWithoutLeakingOrHashing(t *testing.T) { const missingEnv = "PROMPTKIT_PUBLIC_PREPARE_MISSING" const firstKey = "first-direct-key" diff --git a/public_contract_test.go b/public_contract_test.go index 07fd3b9..159b969 100644 --- a/public_contract_test.go +++ b/public_contract_test.go @@ -28,39 +28,94 @@ func TestPreparedRunJSONOmitsZeroTimingValues(t *testing.T) { } func TestBackendIdentityJSONNamesAndOmission(t *testing.T) { - values := []struct { - name string - value any - field string - }{ - {name: "execution target", value: promptkit.ExecutionTarget{BackendID: promptkit.BackendOpenRouter}, field: "backend_id"}, - {name: "prepared run", value: promptkit.PreparedRun{SelectedBackendID: promptkit.BackendOpenRouter}, field: "selected_backend_id"}, - {name: "run result", value: promptkit.RunResult{SelectedBackendID: promptkit.BackendOpenRouter}, field: "selected_backend_id"}, - } - for _, tt := range values { - t.Run(tt.name, func(t *testing.T) { - payload, err := json.Marshal(tt.value) - if err != nil { - t.Fatalf("marshal populated value: %v", err) - } - var object map[string]any - if err := json.Unmarshal(payload, &object); err != nil { - t.Fatalf("decode populated value: %v", err) - } - if object[tt.field] != promptkit.BackendOpenRouter { - t.Fatalf("expected %s=%q, got %s", tt.field, promptkit.BackendOpenRouter, payload) - } - }) - } - - emptyValues := []any{promptkit.ExecutionTarget{}, promptkit.PreparedRun{}, promptkit.RunResult{}} - for _, value := range emptyValues { + t.Run("execution target round trip", func(t *testing.T) { + value := promptkit.ExecutionTarget{BackendID: promptkit.BackendOpenRouter} payload, err := json.Marshal(value) if err != nil { - t.Fatalf("marshal empty value: %v", err) + t.Fatalf("marshal execution target: %v", err) + } + var decoded promptkit.ExecutionTarget + if err := json.Unmarshal(payload, &decoded); err != nil { + t.Fatalf("unmarshal execution target: %v", err) + } + if decoded.BackendID != value.BackendID { + t.Fatalf("backend identity did not round trip: got %q want %q", decoded.BackendID, value.BackendID) + } + }) + + t.Run("prepared run round trip", func(t *testing.T) { + value := promptkit.PreparedRun{SelectedBackendID: promptkit.BackendOpenRouter} + payload, err := json.Marshal(value) + if err != nil { + t.Fatalf("marshal prepared run: %v", err) + } + var decoded promptkit.PreparedRun + if err := json.Unmarshal(payload, &decoded); err != nil { + t.Fatalf("unmarshal prepared run: %v", err) + } + if decoded.SelectedBackendID != value.SelectedBackendID { + t.Fatalf("backend identity did not round trip: got %q want %q", decoded.SelectedBackendID, value.SelectedBackendID) + } + }) + + t.Run("run result round trip", func(t *testing.T) { + value := promptkit.RunResult{SelectedBackendID: promptkit.BackendOpenRouter} + payload, err := json.Marshal(value) + if err != nil { + t.Fatalf("marshal run result: %v", err) + } + var decoded promptkit.RunResult + if err := json.Unmarshal(payload, &decoded); err != nil { + t.Fatalf("unmarshal run result: %v", err) + } + if decoded.SelectedBackendID != value.SelectedBackendID { + t.Fatalf("backend identity did not round trip: got %q want %q", decoded.SelectedBackendID, value.SelectedBackendID) + } + }) + + payload, err := json.Marshal(promptkit.ExecutionTarget{}) + if err != nil { + t.Fatalf("marshal empty execution target: %v", err) + } + if strings.Contains(string(payload), `"backend_id"`) { + t.Fatalf("empty backend identity was not omitted: %s", payload) + } +} + +func TestEndpointOnlyProfileOmitsBackendIdentityFromStableJSON(t *testing.T) { + client := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}} + engine, err := promptkit.NewEngine(promptkit.Config{}, + promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), + promptkit.WithProfiles(promptkit.Profile{ + ID: "profile", Endpoint: "http://example.test/v1", Model: "model", + }), + promptkit.WithLLMClient(client), + ) + if err != nil { + t.Fatalf("construct engine: %v", err) + } + + prepared, err := engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"}) + if err != nil { + t.Fatalf("prepare endpoint-only profile: %v", err) + } + result, err := engine.Run(context.Background(), promptkit.RunRequest{PromptID: "prompt"}) + if err != nil { + t.Fatalf("run endpoint-only profile: %v", err) + } + if prepared.SelectedBackendID != "" || + prepared.EffectiveModelParams.BackendID != "" || + result.SelectedBackendID != "" || + result.EffectiveModelParams.BackendID != "" { + t.Fatalf("endpoint-only profile acquired backend identity: prepared=%+v result=%+v", prepared, result) + } + for _, value := range []any{prepared, result} { + payload, err := json.Marshal(value) + if err != nil { + t.Fatalf("marshal endpoint-only value: %v", err) } if strings.Contains(string(payload), `"backend_id"`) || strings.Contains(string(payload), `"selected_backend_id"`) { - t.Fatalf("empty backend identity was not omitted: %s", payload) + t.Fatalf("endpoint-only backend identity was not omitted: %s", payload) } } } @@ -537,10 +592,14 @@ func TestRepeatedOptionsUseLastValueInEachCategory(t *testing.T) { func TestEngineSupportsConcurrentPrepareAndRun(t *testing.T) { engine, err := promptkit.NewEngine(promptkit.Config{}, promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), - promptkit.WithProfiles(promptkit.Profile{ - ID: "profile", + promptkit.WithBackend(promptkit.Backend{ + ID: "concurrent", Endpoint: "http://example.test/v1", - Model: "model", + }), + promptkit.WithProfiles(promptkit.Profile{ + ID: "profile", + BackendID: "concurrent", + Model: "model", }), promptkit.WithLLMClient(countingLLMClient{}), ) diff --git a/types.go b/types.go index b44e63a..8767ad8 100644 --- a/types.go +++ b/types.go @@ -101,8 +101,8 @@ type RunRequest struct { // Vars supplies Go-template data for messages and the session ID. Nil and // empty maps are equivalent. Vars map[string]string - // Execution optionally overrides individual profile execution settings. - // Nil uses the selected profile over framework defaults. + // Execution optionally overrides individual execution settings. Nil uses + // the selected profile over its backend, when any, and framework defaults. Execution *ExecutionTargetOverride // Validation optionally replaces the prompt's complete output contract. It // does not merge individual fields. Nil uses the prompt contract. @@ -126,8 +126,8 @@ type PreparedRun struct { // SelectedProfileID is the explicit request profile or prompt default that // supplied execution settings. SelectedProfileID string `json:"selected_profile_id"` - // SelectedBackendID is the selected profile's normalized backend ID. It is - // empty for an endpoint-only profile. + // SelectedBackendID equals EffectiveModelParams.BackendID. It is empty for + // an endpoint-only profile. SelectedBackendID string `json:"selected_backend_id,omitempty"` // EffectiveModelParams contains framework defaults overlaid by the selected // backend, profile, and then request overrides. It excludes resolved API-key @@ -184,8 +184,8 @@ type RunResult struct { RenderedPromptHash string `json:"rendered_prompt_hash"` // SelectedProfileID identifies the profile used for execution. SelectedProfileID string `json:"selected_profile_id"` - // SelectedBackendID is the selected profile's normalized backend ID. It is - // empty for an endpoint-only profile. + // SelectedBackendID equals EffectiveModelParams.BackendID. It is empty for + // an endpoint-only profile. SelectedBackendID string `json:"selected_backend_id,omitempty"` // ModelName is the effective model name and equals // EffectiveModelParams.Model. @@ -268,7 +268,8 @@ type ArtifactReader interface { type ExecutionTarget struct { // BackendID is the effective routing identity selected by the profile. It // remains unchanged when a profile or request overrides Endpoint and is - // empty for endpoint-only profiles. + // empty for endpoint-only profiles. It is supplied to injected LLMClient + // implementations as part of the effective target. BackendID string `json:"backend_id,omitempty"` // Endpoint is the model-provider base URL. Endpoint string `json:"endpoint"` @@ -298,13 +299,15 @@ type ExecutionTarget struct { // ExecutionTargetOverride represents per-request runtime setting overrides and // has no stable JSON representation. // -// Non-empty string fields replace profile values. Non-nil numeric pointers -// replace profile values and preserve explicit zero. A non-empty ExtraParams -// map replaces the complete profile map rather than merging keys. Empty string -// fields, nil pointers, and a nil or empty ExtraParams map inherit the selected -// profile over framework defaults. +// Non-empty string fields replace profile and backend values. Non-nil numeric +// pointers replace profile values and preserve explicit zero. A non-empty +// ExtraParams map replaces the complete profile or backend map rather than +// merging keys. Empty string fields, nil pointers, and a nil or empty +// ExtraParams map inherit the selected profile over its backend, when any, and +// framework defaults. type ExecutionTargetOverride struct { - // Endpoint replaces the profile endpoint when non-empty. + // Endpoint replaces the profile or backend endpoint when non-empty without + // changing the effective BackendID. Endpoint string // Model replaces the profile model when non-empty. Model string @@ -322,12 +325,13 @@ type ExecutionTargetOverride struct { // ReasoningEffort replaces the profile value when non-blank. An empty value // cannot clear a profile setting. ReasoningEffort string - // APIKeyEnv replaces the profile environment-variable name when non-blank. - // A direct RunRequest.APIKey still takes precedence over environment lookup. + // APIKeyEnv replaces the profile or backend environment-variable name when + // non-blank. A direct RunRequest.APIKey still takes precedence over + // environment lookup. APIKeyEnv string - // ExtraParams, when non-empty, replaces the profile map. Values must be - // JSON-compatible: nil, booleans, finite numbers, strings, arrays or slices, - // and maps with non-empty string keys. Cycles are invalid. + // ExtraParams, when non-empty, replaces the complete profile or backend map. + // Values must be JSON-compatible: nil, booleans, finite numbers, strings, + // arrays or slices, and maps with non-empty string keys. Cycles are invalid. ExtraParams map[string]any } @@ -346,7 +350,8 @@ type Profile struct { // ID is the required non-blank profile identifier. WithProfiles trims it. ID string // BackendID optionally selects an engine backend. WithProfiles trims it. - // Backend membership is checked when a request selects the profile. + // Backend membership is checked when a request selects the profile; an + // unknown ID makes preparation fail with ErrProfileLoad. BackendID string // Endpoint is the model-provider base URL. It is required only when // BackendID is blank and otherwise overrides the backend endpoint when @@ -373,8 +378,8 @@ type Profile struct { // supplies ExecutionTargetOverride.APIKeyEnv. It does not store a credential. APIKeyRequired bool // ExtraParams contains provider-specific JSON-compatible values. An empty - // map inherits framework defaults. WithProfiles validates and deeply copies - // it during NewEngine. + // map inherits backend request defaults, when any. WithProfiles validates + // and deeply copies it during NewEngine. ExtraParams map[string]any }