From 810f80e7c9111f814d7bb46facb576e96144c81d Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Wed, 29 Jul 2026 17:10:34 +0000 Subject: [PATCH] Resolve profiles through built-in backends --- convert.go | 3 + docs/formats.md | 32 +- docs/internal/runner.md | 47 +-- docs/internal/sources.md | 14 +- docs/roadmap/implementation.md | 2 + engine.go | 7 + engine_test.go | 17 +- internal/domain/domain.go | 4 + .../builtin/assets/aion-labs/aion-2.yml | 3 +- .../assets/anthropic/claude-fable-latest.yml | 3 +- .../assets/anthropic/claude-haiku-latest.yml | 3 +- .../assets/anthropic/claude-opus-latest.yml | 3 +- .../assets/anthropic/claude-sonnet-latest.yml | 3 +- .../builtin/assets/deepseek/deepseek-3-2.yml | 3 +- .../assets/deepseek/deepseek-4-flash.yml | 3 +- .../assets/deepseek/deepseek-4-pro.yml | 3 +- .../assets/google/gemini-2-flash-lite.yml | 3 +- .../builtin/assets/google/gemini-2-flash.yml | 3 +- .../builtin/assets/google/gemini-2-pro.yml | 3 +- .../assets/google/gemini-3-flash-lite.yml | 3 +- .../assets/google/gemini-flash-latest.yml | 3 +- .../assets/google/gemini-pro-latest.yml | 3 +- .../builtin/assets/google/gemma-4-31b.yml | 3 +- .../builtin/assets/minimax/minimax-m2.yml | 3 +- .../builtin/assets/minimax/minimax-m3.yml | 3 +- .../assets/mistral/mistral-large-2512.yml | 3 +- .../assets/mistral/mistral-medium-3-5.yml | 3 +- .../assets/mistral/mistral-small-3.yml | 3 +- .../assets/mistral/mistral-small-4.yml | 3 +- .../assets/nvidia/nemotron-3-ultra.yml | 3 +- .../builtin/assets/openai/gpt-5-mini.yml | 3 +- .../builtin/assets/openai/gpt-5-nano.yml | 3 +- internal/profile/builtin/repository_test.go | 16 + internal/profile/filesystem_repository.go | 5 +- internal/profile/repository_test.go | 37 +++ internal/usecase/runner.go | 54 +++- internal/usecase/runner_test.go | 288 +++++++++++++----- json.go | 5 + profiles.go | 13 +- public_contract_test.go | 57 ++++ types.go | 38 ++- 41 files changed, 538 insertions(+), 173 deletions(-) diff --git a/convert.go b/convert.go index 6cb3264..745f913 100644 --- a/convert.go +++ b/convert.go @@ -32,6 +32,7 @@ func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun { PromptVersion: prepared.PromptVersion, PromptHash: prepared.PromptHash, SelectedProfileID: prepared.SelectedProfileID, + SelectedBackendID: prepared.SelectedBackendID, EffectiveModelParams: fromDomainExecutionTarget(prepared.EffectiveModelParams), OutputContract: fromDomainOutputContract(prepared.OutputContract), StructuredOutput: fromDomainStructuredOutputSpec(prepared.StructuredOutput), @@ -59,6 +60,7 @@ func fromDomainRunResult(result *domain.RunResult) *RunResult { PromptHash: result.PromptHash, RenderedPromptHash: result.RenderedPromptHash, SelectedProfileID: result.SelectedProfileID, + SelectedBackendID: result.SelectedBackendID, ModelName: result.ModelName, Endpoint: result.Endpoint, EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams), @@ -151,6 +153,7 @@ func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget { return ExecutionTarget{ + BackendID: target.BackendID, Endpoint: target.Endpoint, Model: target.Model, Temperature: target.Temperature, diff --git a/docs/formats.md b/docs/formats.md index 5ae6c1e..1e4ce2a 100644 --- a/docs/formats.md +++ b/docs/formats.md @@ -136,7 +136,7 @@ A profile supplies model execution settings: ```yaml id: local-summary -endpoint: http://localhost:8000/v1 +backend: openrouter model: example-model temperature: 0.2 max_tokens: 500 @@ -144,7 +144,6 @@ top_p: 0.95 timeout_seconds: 90 service_tier: flex reasoning_effort: medium -api_key_env: EXAMPLE_API_KEY extra_params: provider_option: enabled ``` @@ -152,7 +151,8 @@ extra_params: | Field | Required | Meaning | | --- | --- | --- | | `id` | yes | Non-empty profile identifier. IDs must be unique within one source. | -| `endpoint` | yes | Non-empty OpenAI-compatible base URL, including an API version path when required. | +| `backend` | unless `endpoint` is present | Backend registry ID. It is trimmed and registry membership is checked when the profile is prepared. | +| `endpoint` | unless `backend` is present | Non-empty OpenAI-compatible base URL, including an API version path when required. When both connection fields are present, this overrides the backend endpoint without changing backend identity. | | `model` | yes | Non-empty provider model name. | | `temperature` | no | Number from 0 through 2. | | `max_tokens` | no | Integer zero or greater. | @@ -166,6 +166,9 @@ extra_params: Raw `api_key` is prohibited in profile YAML. Store only an environment variable name in `api_key_env`. +Promptkit does not infer a backend from a model or endpoint. Endpoint-only +profiles remain supported and have no effective backend ID. + `extra_params` accepts null, booleans, finite numbers, strings, arrays, and objects with string keys. Keys must be non-empty. With the built-in client, they also cannot collide with the standard fields listed in the @@ -176,8 +179,9 @@ they also cannot collide with the standard fields listed in the Execution settings resolve in this order: 1. framework defaults; -2. the selected profile; and -3. request `ExecutionTargetOverride` values. +2. the selected backend, when the profile names one; +3. the selected profile; and +4. request `ExecutionTargetOverride` values. The framework defaults are: @@ -194,8 +198,10 @@ explicit zero is preserved. In particular, an explicit request `timeout_seconds` of zero disables the per-generation deadline while leaving the caller context and transport timeout intact. -Non-empty request strings replace profile strings. A non-empty request -`ExtraParams` map replaces the profile map rather than merging keys. +Non-empty profile strings replace backend defaults, and non-empty request +strings replace both. Backend identity is retained when either layer overrides +the endpoint. A non-empty `extra_params` map at each layer replaces the entire +lower-precedence map rather than merging keys. The [outbound integration contract](integrations/openai-compatible-chat.md) defines how the effective settings are serialized. @@ -217,9 +223,11 @@ invalid matching profile is an error and does not fall back. In-memory ## Built-In Profile Catalog -Built-ins use the OpenRouter-compatible endpoint and -`OPENROUTER_API_KEY`. A custom or in-memory profile with the same ID takes -precedence. +Every built-in selects the `openrouter` backend. The engine's built-in backend +registry supplies `https://openrouter.ai/api/v1` and the environment-variable +name `OPENROUTER_API_KEY`, so individual profiles contain only model and +generation settings. A custom or in-memory profile with the same profile ID +takes precedence. | Provider | ID | Model | | --- | --- | --- | @@ -270,6 +278,10 @@ prompt, profile, schema, or example files: - a request can provide a direct `APIKey` or override `APIKeyEnv`; and - a direct request key takes precedence over environment lookup. +After a direct request key, the credential-source precedence is request +`APIKeyEnv`, profile `api_key_env`, then the backend default. An in-memory +profile with `APIKeyRequired` clears an inherited backend environment name and +requires a direct key unless the request explicitly supplies `APIKeyEnv`. Promptkit validates required credential availability during preparation. Direct keys are excluded from JSON results and redacted by public string formatters. Environment-variable names may appear in prepared metadata, but diff --git a/docs/internal/runner.md b/docs/internal/runner.md index 8bfdb34..0b51110 100644 --- a/docs/internal/runner.md +++ b/docs/internal/runner.md @@ -17,10 +17,11 @@ and override semantics consumed by the runner. ## Collaborators `Runner` coordinates narrow internal interfaces for prompt definitions, -profiles, artifacts, rendering, model generation, and validation. Schema -documents are loaded through the validator's optional schema-loader interface. -An output repairer can be injected internally, but the ordinary runner -constructor does not enable one. +profiles, backend resolution, artifacts, rendering, model generation, and +validation. The root engine supplies an immutable built-in backend registry. +Schema documents are loaded through the validator's optional schema-loader +interface. An output repairer can be injected internally, but the ordinary +runner constructor does not enable one. Each invocation carries its state in request, prepared-run, and result values. The runner has no durable run or session store. @@ -32,20 +33,25 @@ The runner has no durable run or session store. 1. validate the prompt selection and load the prompt definition; 2. hash the loaded definition; 3. select the request profile or the prompt's default profile; -4. resolve application-neutral defaults, profile values, and explicit request - overrides in that order; -5. validate endpoint, model, numeric overrides, and credential requirements; -6. resolve the output contract and load a structured-output schema when +4. resolve the profile's backend ID, when present; +5. resolve application-neutral defaults, backend defaults, profile values, + and explicit request overrides in that order; +6. validate endpoint, model, numeric overrides, and credential requirements; +7. resolve the output contract and load a structured-output schema when required; -7. load and hash input artifacts; -8. render and hash the prompt; and -9. return the effective settings, source identities, messages, hashes, and +8. load and hash input artifacts; +9. render and hash the prompt; and +10. return the effective settings, source identities, messages, hashes, and preparation timing. Pointer-based numeric overrides preserve an explicit zero. Invalid negative or -out-of-range values fail as invalid requests. A direct API key takes -precedence over environment lookup for execution; secret values remain -excluded from serialized metadata. +out-of-range values fail as invalid requests. Endpoint overrides do not change +the selected backend identity. Non-empty extra-parameter maps replace whole +lower-precedence maps. A direct API key takes precedence over environment +lookup; otherwise request, profile, and backend environment-variable names +apply in that order. A profile requiring a direct key clears an inherited +backend environment name unless the request supplies its own name. Secret +values remain excluded from serialized metadata. ## Run Flow @@ -60,8 +66,10 @@ target, validation errors, prior output, and structured-output specification. This capability remains internal and is not a public option. A successful result includes the output artifact and raw output, validation -state, prompt and rendered-prompt hashes, selected profile, effective settings, -input hashes, token usage, a generated run identifier, and UTC timing. +state, prompt and rendered-prompt hashes, selected profile and backend, +effective settings, input hashes, token usage, a generated run identifier, and +UTC timing. The same effective target, including backend identity, reaches +generation and any repair attempt. ## Failure Categories @@ -71,13 +79,16 @@ validation failures. Wrapping preserves the package identities mapped by the public facade and retains collaborator identities where they are part of the internal contract. Context cancellation propagates through the invoked collaborator and is classified by the owning operation. +An unknown selected backend, or a selected backend with no configured resolver, +is classified as a profile-load failure. ## Test Ownership And Changes The [runner tests](../../internal/usecase/runner_test.go) own preparation order, selection and override precedence, schema-before-generation behavior, hashing, -generation and validation outcomes, bounded repair, credentials and redaction, -error categories, artifact metadata, usage, and timing. +generation and validation outcomes, backend propagation, bounded repair, +credentials and redaction, error categories, artifact metadata, usage, and +timing. Changes to orchestration should continue to use the existing package interfaces, keep request state local to an invocation, and preserve `Run`'s use diff --git a/docs/internal/sources.md b/docs/internal/sources.md index 6f5fda6..33d22db 100644 --- a/docs/internal/sources.md +++ b/docs/internal/sources.md @@ -24,13 +24,19 @@ duplicate detection, and source containment: `internal/profile` loads and validates execution profiles from an operating-system filesystem or an `fs.FS`. It supports a primary repository -with fallback only when the primary reports that a profile is absent. +with fallback only when the primary reports that a profile is absent. Strict +YAML decoding recognizes the optional `backend` field, trims its value, and +requires a model plus at least one non-blank backend or endpoint. Loading does +not check registry membership because the available registry belongs to the +assembled engine; the runner checks membership during preparation. `internal/profile/builtin` embeds the maintained built-in profile catalog and -can place a caller-selected repository ahead of that catalog. Profile behavior -is owned by the +can place a caller-selected repository ahead of that catalog. Every embedded +profile selects `openrouter` and inherits its endpoint and credential +environment-variable name from the built-in backend registry rather than +repeating those values. Profile behavior is owned by the [profile repository tests](../../internal/profile/repository_test.go), while -catalog completeness, duplicate IDs, and overlay behavior are owned by the +catalog completeness, backend-selection invariant, duplicate IDs, and overlay behavior are owned by the [built-in repository tests](../../internal/profile/builtin/repository_test.go). ## Ordinary Artifacts diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index ba62d62..3e8b176 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -299,6 +299,8 @@ immutable and the final repository race suite exercises concurrent reads. ## Stage 2 — Profile Selection And Effective Resolution +**Status:** Complete. + ### Goal Make the built-in OpenRouter backend selectable by file and in-memory profiles, diff --git a/engine.go b/engine.go index 063f576..9548534 100644 --- a/engine.go +++ b/engine.go @@ -12,6 +12,7 @@ import ( "time" artifactadapter "gitea.maximumdirect.net/eric/promptkit/internal/artifact" + "gitea.maximumdirect.net/eric/promptkit/internal/backend" "gitea.maximumdirect.net/eric/promptkit/internal/defaults" "gitea.maximumdirect.net/eric/promptkit/internal/llm" "gitea.maximumdirect.net/eric/promptkit/internal/profile" @@ -335,6 +336,11 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) { profiles = profile.NewOverlayRepository(options.memoryProfiles, profiles) } + backendRegistry, err := backend.NewRegistry(nil) + if err != nil { + return nil, fmt.Errorf("%w: failed to construct backend registry: %v", ErrInvalidConfig, err) + } + validator := options.validator if !options.validatorSource { schemaDir := cfg.SchemaDir @@ -365,6 +371,7 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) { runner: usecase.NewRunner( promptDefs, profiles, + backendRegistry, artifacts, prompt.NewGoRenderer(), llmClient, diff --git a/engine_test.go b/engine_test.go index c20338c..93d2c88 100644 --- a/engine_test.go +++ b/engine_test.go @@ -1271,6 +1271,18 @@ func TestPrepareUsesBuiltInProfileWithoutProfileDir(t *testing.T) { if prepared.SelectedProfileID != "mistral-small-3" { t.Fatalf("unexpected selected profile: %q", prepared.SelectedProfileID) } + if prepared.SelectedBackendID != promptkit.BackendOpenRouter { + t.Fatalf("unexpected selected backend: %q", prepared.SelectedBackendID) + } + if prepared.EffectiveModelParams.BackendID != promptkit.BackendOpenRouter { + t.Fatalf("unexpected effective backend: %q", prepared.EffectiveModelParams.BackendID) + } + if prepared.EffectiveModelParams.Endpoint != "https://openrouter.ai/api/v1" { + t.Fatalf("unexpected built-in endpoint: %q", prepared.EffectiveModelParams.Endpoint) + } + if prepared.EffectiveModelParams.APIKeyEnv != "OPENROUTER_API_KEY" { + t.Fatalf("unexpected built-in api key environment name: %q", prepared.EffectiveModelParams.APIKeyEnv) + } if prepared.EffectiveModelParams.Model != "mistralai/mistral-small-3.2-24b-instruct" { t.Fatalf("unexpected built-in model: %q", prepared.EffectiveModelParams.Model) } @@ -1641,6 +1653,7 @@ func TestOpenAICompatibleProfileRunsThroughNormalProfilePath(t *testing.T) { fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}} prof := promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ ID: "template-profile", + BackendID: " openrouter ", Endpoint: "http://template/v1", Model: "template-model", APIKeyRequired: true, @@ -1672,7 +1685,9 @@ func TestOpenAICompatibleProfileRunsThroughNormalProfilePath(t *testing.T) { if len(fake.requests) != 1 { t.Fatalf("expected one request, got %d", len(fake.requests)) } - if fake.requests[0].Target.Model != "template-model" || fake.requests[0].APIKey != "template-key" { + if fake.requests[0].Target.BackendID != promptkit.BackendOpenRouter || + fake.requests[0].Target.Model != "template-model" || + fake.requests[0].APIKey != "template-key" { t.Fatalf("unexpected generated request: %+v", fake.requests[0]) } if !reflect.DeepEqual(fake.requests[0].Target.ExtraParams, map[string]any{"provider": "template"}) { diff --git a/internal/domain/domain.go b/internal/domain/domain.go index d1dfd3d..3caefad 100644 --- a/internal/domain/domain.go +++ b/internal/domain/domain.go @@ -81,6 +81,7 @@ type RunResult struct { PromptHash string RenderedPromptHash string SelectedProfileID string + SelectedBackendID string ModelName string Endpoint string EffectiveModelParams ExecutionTarget @@ -98,6 +99,7 @@ type PreparedRun struct { PromptVersion string `json:"prompt_version,omitempty"` PromptHash string `json:"prompt_hash,omitempty"` SelectedProfileID string `json:"selected_profile_id"` + SelectedBackendID string `json:"selected_backend_id,omitempty"` EffectiveModelParams ExecutionTarget `json:"effective_model_params"` TargetPresence ExecutionTargetPresence `json:"-"` OutputContract OutputContract `json:"output_contract"` @@ -168,6 +170,7 @@ type Backend struct { // ExecutionProfile describes how and where to execute a model. type ExecutionProfile struct { ID string `yaml:"id"` + BackendID string `yaml:"backend"` Endpoint string `yaml:"endpoint"` Model string `yaml:"model"` Temperature float64 `yaml:"temperature"` @@ -206,6 +209,7 @@ type ExecutionTargetPresence struct { // ExecutionTarget represents effective model runtime settings for a run. type ExecutionTarget struct { + BackendID string `yaml:"backend" json:"backend_id,omitempty"` Endpoint string `yaml:"endpoint" json:"endpoint"` Model string `yaml:"model" json:"model"` Temperature float64 `yaml:"temperature" json:"temperature"` diff --git a/internal/profile/builtin/assets/aion-labs/aion-2.yml b/internal/profile/builtin/assets/aion-labs/aion-2.yml index 785a6fd..a22cd8f 100644 --- a/internal/profile/builtin/assets/aion-labs/aion-2.yml +++ b/internal/profile/builtin/assets/aion-labs/aion-2.yml @@ -1,9 +1,8 @@ id: aion-2 -endpoint: https://openrouter.ai/api/v1 +backend: openrouter model: aion-labs/aion-2.0 temperature: 0.72 reasoning_effort: high top_p: 0.95 timeout_seconds: 180 -api_key_env: OPENROUTER_API_KEY service_tier: flex diff --git a/internal/profile/builtin/assets/anthropic/claude-fable-latest.yml b/internal/profile/builtin/assets/anthropic/claude-fable-latest.yml index 1a1e7aa..0affde6 100644 --- a/internal/profile/builtin/assets/anthropic/claude-fable-latest.yml +++ b/internal/profile/builtin/assets/anthropic/claude-fable-latest.yml @@ -1,7 +1,6 @@ id: claude-fable-latest -endpoint: https://openrouter.ai/api/v1 +backend: openrouter model: "~anthropic/claude-fable-latest" reasoning_effort: high timeout_seconds: 600 -api_key_env: OPENROUTER_API_KEY service_tier: flex diff --git a/internal/profile/builtin/assets/anthropic/claude-haiku-latest.yml b/internal/profile/builtin/assets/anthropic/claude-haiku-latest.yml index f22bf15..e5ada35 100644 --- a/internal/profile/builtin/assets/anthropic/claude-haiku-latest.yml +++ b/internal/profile/builtin/assets/anthropic/claude-haiku-latest.yml @@ -1,7 +1,6 @@ id: claude-haiku-latest -endpoint: https://openrouter.ai/api/v1 +backend: openrouter model: "~anthropic/claude-haiku-latest" reasoning_effort: medium timeout_seconds: 240 -api_key_env: OPENROUTER_API_KEY service_tier: flex diff --git a/internal/profile/builtin/assets/anthropic/claude-opus-latest.yml b/internal/profile/builtin/assets/anthropic/claude-opus-latest.yml index c192c3a..6d628ac 100644 --- a/internal/profile/builtin/assets/anthropic/claude-opus-latest.yml +++ b/internal/profile/builtin/assets/anthropic/claude-opus-latest.yml @@ -1,7 +1,6 @@ id: claude-opus-latest -endpoint: https://openrouter.ai/api/v1 +backend: openrouter model: "~anthropic/claude-opus-latest" reasoning_effort: high timeout_seconds: 240 -api_key_env: OPENROUTER_API_KEY service_tier: flex diff --git a/internal/profile/builtin/assets/anthropic/claude-sonnet-latest.yml b/internal/profile/builtin/assets/anthropic/claude-sonnet-latest.yml index c7be449..adce8d6 100644 --- a/internal/profile/builtin/assets/anthropic/claude-sonnet-latest.yml +++ b/internal/profile/builtin/assets/anthropic/claude-sonnet-latest.yml @@ -1,7 +1,6 @@ id: claude-sonnet-latest -endpoint: https://openrouter.ai/api/v1 +backend: openrouter model: "~anthropic/claude-sonnet-latest" reasoning_effort: high timeout_seconds: 240 -api_key_env: OPENROUTER_API_KEY service_tier: flex diff --git a/internal/profile/builtin/assets/deepseek/deepseek-3-2.yml b/internal/profile/builtin/assets/deepseek/deepseek-3-2.yml index 1f27fbc..b857d0a 100644 --- a/internal/profile/builtin/assets/deepseek/deepseek-3-2.yml +++ b/internal/profile/builtin/assets/deepseek/deepseek-3-2.yml @@ -1,7 +1,6 @@ id: deepseek-3-2 -endpoint: https://openrouter.ai/api/v1 +backend: openrouter model: deepseek/deepseek-v3.2 reasoning_effort: high timeout_seconds: 180 -api_key_env: OPENROUTER_API_KEY service_tier: flex diff --git a/internal/profile/builtin/assets/deepseek/deepseek-4-flash.yml b/internal/profile/builtin/assets/deepseek/deepseek-4-flash.yml index a2edf84..bffa903 100644 --- a/internal/profile/builtin/assets/deepseek/deepseek-4-flash.yml +++ b/internal/profile/builtin/assets/deepseek/deepseek-4-flash.yml @@ -1,7 +1,6 @@ id: deepseek-4-flash -endpoint: https://openrouter.ai/api/v1 +backend: openrouter model: deepseek/deepseek-v4-flash #reasoning_effort: medium timeout_seconds: 180 -api_key_env: OPENROUTER_API_KEY service_tier: flex diff --git a/internal/profile/builtin/assets/deepseek/deepseek-4-pro.yml b/internal/profile/builtin/assets/deepseek/deepseek-4-pro.yml index c7af1ff..ec79389 100644 --- a/internal/profile/builtin/assets/deepseek/deepseek-4-pro.yml +++ b/internal/profile/builtin/assets/deepseek/deepseek-4-pro.yml @@ -1,7 +1,6 @@ id: deepseek-4-pro -endpoint: https://openrouter.ai/api/v1 +backend: openrouter model: deepseek/deepseek-v4-pro reasoning_effort: high timeout_seconds: 180 -api_key_env: OPENROUTER_API_KEY service_tier: flex diff --git a/internal/profile/builtin/assets/google/gemini-2-flash-lite.yml b/internal/profile/builtin/assets/google/gemini-2-flash-lite.yml index e648e70..970f29e 100644 --- a/internal/profile/builtin/assets/google/gemini-2-flash-lite.yml +++ b/internal/profile/builtin/assets/google/gemini-2-flash-lite.yml @@ -1,9 +1,8 @@ id: gemini-2-flash-lite -endpoint: https://openrouter.ai/api/v1 +backend: openrouter model: "google/gemini-2.5-flash-lite" #temperature: 0.15 reasoning_effort: high #top_p: 0.98 timeout_seconds: 240 -api_key_env: OPENROUTER_API_KEY service_tier: flex diff --git a/internal/profile/builtin/assets/google/gemini-2-flash.yml b/internal/profile/builtin/assets/google/gemini-2-flash.yml index 3b1267e..01bd8d7 100644 --- a/internal/profile/builtin/assets/google/gemini-2-flash.yml +++ b/internal/profile/builtin/assets/google/gemini-2-flash.yml @@ -1,9 +1,8 @@ id: gemini-2-flash -endpoint: https://openrouter.ai/api/v1 +backend: openrouter model: "google/gemini-2.5-flash" #temperature: 0.15 reasoning_effort: high #top_p: 0.98 timeout_seconds: 240 -api_key_env: OPENROUTER_API_KEY service_tier: flex diff --git a/internal/profile/builtin/assets/google/gemini-2-pro.yml b/internal/profile/builtin/assets/google/gemini-2-pro.yml index b779db6..fed78cd 100644 --- a/internal/profile/builtin/assets/google/gemini-2-pro.yml +++ b/internal/profile/builtin/assets/google/gemini-2-pro.yml @@ -1,9 +1,8 @@ id: gemini-2-pro -endpoint: https://openrouter.ai/api/v1 +backend: openrouter model: "google/gemini-2.5-pro" #temperature: 0.15 reasoning_effort: high #top_p: 0.98 timeout_seconds: 240 -api_key_env: OPENROUTER_API_KEY service_tier: flex diff --git a/internal/profile/builtin/assets/google/gemini-3-flash-lite.yml b/internal/profile/builtin/assets/google/gemini-3-flash-lite.yml index ae44f66..4ff439b 100644 --- a/internal/profile/builtin/assets/google/gemini-3-flash-lite.yml +++ b/internal/profile/builtin/assets/google/gemini-3-flash-lite.yml @@ -1,9 +1,8 @@ id: gemini-3-flash-lite -endpoint: https://openrouter.ai/api/v1 +backend: openrouter model: "google/gemini-3.1-flash-lite" #temperature: 0.15 reasoning_effort: high #top_p: 0.98 timeout_seconds: 240 -api_key_env: OPENROUTER_API_KEY service_tier: flex diff --git a/internal/profile/builtin/assets/google/gemini-flash-latest.yml b/internal/profile/builtin/assets/google/gemini-flash-latest.yml index 2bcda6e..89acd90 100644 --- a/internal/profile/builtin/assets/google/gemini-flash-latest.yml +++ b/internal/profile/builtin/assets/google/gemini-flash-latest.yml @@ -1,9 +1,8 @@ id: gemini-flash-latest -endpoint: https://openrouter.ai/api/v1 +backend: openrouter model: "~google/gemini-flash-latest" #temperature: 0.15 reasoning_effort: high #top_p: 0.98 timeout_seconds: 240 -api_key_env: OPENROUTER_API_KEY service_tier: flex diff --git a/internal/profile/builtin/assets/google/gemini-pro-latest.yml b/internal/profile/builtin/assets/google/gemini-pro-latest.yml index 2e77cbc..2ce5303 100644 --- a/internal/profile/builtin/assets/google/gemini-pro-latest.yml +++ b/internal/profile/builtin/assets/google/gemini-pro-latest.yml @@ -1,9 +1,8 @@ id: gemini-pro-latest -endpoint: https://openrouter.ai/api/v1 +backend: openrouter model: "~google/gemini-pro-latest" #temperature: 0.15 reasoning_effort: high #top_p: 0.98 timeout_seconds: 240 -api_key_env: OPENROUTER_API_KEY service_tier: flex diff --git a/internal/profile/builtin/assets/google/gemma-4-31b.yml b/internal/profile/builtin/assets/google/gemma-4-31b.yml index f8ff113..39c3529 100644 --- a/internal/profile/builtin/assets/google/gemma-4-31b.yml +++ b/internal/profile/builtin/assets/google/gemma-4-31b.yml @@ -1,9 +1,8 @@ id: gemma-4-31b -endpoint: https://openrouter.ai/api/v1 +backend: openrouter model: google/gemma-4-31b-it:exacto temperature: 0.15 reasoning_effort: high top_p: 0.98 timeout_seconds: 240 -api_key_env: OPENROUTER_API_KEY service_tier: flex diff --git a/internal/profile/builtin/assets/minimax/minimax-m2.yml b/internal/profile/builtin/assets/minimax/minimax-m2.yml index 6ce7fdf..385a181 100644 --- a/internal/profile/builtin/assets/minimax/minimax-m2.yml +++ b/internal/profile/builtin/assets/minimax/minimax-m2.yml @@ -1,9 +1,8 @@ id: minimax-m2 -endpoint: https://openrouter.ai/api/v1 +backend: openrouter model: minimax/minimax-m2.5 temperature: 0.5 reasoning_effort: high top_p: 0.95 timeout_seconds: 180 -api_key_env: OPENROUTER_API_KEY service_tier: flex diff --git a/internal/profile/builtin/assets/minimax/minimax-m3.yml b/internal/profile/builtin/assets/minimax/minimax-m3.yml index da9a3af..c0c1bd0 100644 --- a/internal/profile/builtin/assets/minimax/minimax-m3.yml +++ b/internal/profile/builtin/assets/minimax/minimax-m3.yml @@ -1,9 +1,8 @@ id: minimax-m3 -endpoint: https://openrouter.ai/api/v1 +backend: openrouter model: minimax/minimax-m3 #temperature: 0.5 reasoning_effort: high #top_p: 0.95 timeout_seconds: 180 -api_key_env: OPENROUTER_API_KEY service_tier: flex diff --git a/internal/profile/builtin/assets/mistral/mistral-large-2512.yml b/internal/profile/builtin/assets/mistral/mistral-large-2512.yml index 106ea2e..c77c9d8 100644 --- a/internal/profile/builtin/assets/mistral/mistral-large-2512.yml +++ b/internal/profile/builtin/assets/mistral/mistral-large-2512.yml @@ -1,7 +1,6 @@ id: mistral-large-2512 -endpoint: https://openrouter.ai/api/v1 +backend: openrouter model: mistralai/mistral-large-2512 temperature: 0.15 top_p: 0.98 timeout_seconds: 180 -api_key_env: OPENROUTER_API_KEY diff --git a/internal/profile/builtin/assets/mistral/mistral-medium-3-5.yml b/internal/profile/builtin/assets/mistral/mistral-medium-3-5.yml index 762149f..b5201ca 100644 --- a/internal/profile/builtin/assets/mistral/mistral-medium-3-5.yml +++ b/internal/profile/builtin/assets/mistral/mistral-medium-3-5.yml @@ -1,8 +1,7 @@ id: mistral-medium-3-5 -endpoint: https://openrouter.ai/api/v1 +backend: openrouter model: mistralai/mistral-medium-3-5 temperature: 0.15 reasoning_effort: high top_p: 0.98 timeout_seconds: 180 -api_key_env: OPENROUTER_API_KEY diff --git a/internal/profile/builtin/assets/mistral/mistral-small-3.yml b/internal/profile/builtin/assets/mistral/mistral-small-3.yml index d077918..7e01e55 100644 --- a/internal/profile/builtin/assets/mistral/mistral-small-3.yml +++ b/internal/profile/builtin/assets/mistral/mistral-small-3.yml @@ -1,7 +1,6 @@ id: mistral-small-3 -endpoint: https://openrouter.ai/api/v1 +backend: openrouter model: mistralai/mistral-small-3.2-24b-instruct temperature: 0.05 top_p: 1.0 timeout_seconds: 180 -api_key_env: OPENROUTER_API_KEY diff --git a/internal/profile/builtin/assets/mistral/mistral-small-4.yml b/internal/profile/builtin/assets/mistral/mistral-small-4.yml index fd0fce6..2d9b44c 100644 --- a/internal/profile/builtin/assets/mistral/mistral-small-4.yml +++ b/internal/profile/builtin/assets/mistral/mistral-small-4.yml @@ -1,8 +1,7 @@ id: mistral-small-4 -endpoint: https://openrouter.ai/api/v1 +backend: openrouter model: mistralai/mistral-small-2603 temperature: 0.1 reasoning_effort: high top_p: 0.98 timeout_seconds: 180 -api_key_env: OPENROUTER_API_KEY diff --git a/internal/profile/builtin/assets/nvidia/nemotron-3-ultra.yml b/internal/profile/builtin/assets/nvidia/nemotron-3-ultra.yml index bb55536..90f84f3 100644 --- a/internal/profile/builtin/assets/nvidia/nemotron-3-ultra.yml +++ b/internal/profile/builtin/assets/nvidia/nemotron-3-ultra.yml @@ -1,7 +1,6 @@ id: nemotron-3-ultra -endpoint: https://openrouter.ai/api/v1 +backend: openrouter model: nvidia/nemotron-3-ultra-550b-a55b reasoning_effort: high timeout_seconds: 180 -api_key_env: OPENROUTER_API_KEY service_tier: flex diff --git a/internal/profile/builtin/assets/openai/gpt-5-mini.yml b/internal/profile/builtin/assets/openai/gpt-5-mini.yml index 72ea4b7..c9b4bb2 100644 --- a/internal/profile/builtin/assets/openai/gpt-5-mini.yml +++ b/internal/profile/builtin/assets/openai/gpt-5-mini.yml @@ -1,7 +1,6 @@ id: gpt-5-mini -endpoint: https://openrouter.ai/api/v1 +backend: openrouter model: "openai/gpt-5.4-mini" reasoning_effort: high timeout_seconds: 240 -api_key_env: OPENROUTER_API_KEY service_tier: flex diff --git a/internal/profile/builtin/assets/openai/gpt-5-nano.yml b/internal/profile/builtin/assets/openai/gpt-5-nano.yml index 069a3d5..0688ddc 100644 --- a/internal/profile/builtin/assets/openai/gpt-5-nano.yml +++ b/internal/profile/builtin/assets/openai/gpt-5-nano.yml @@ -1,7 +1,6 @@ id: gpt-5-nano -endpoint: https://openrouter.ai/api/v1 +backend: openrouter model: "openai/gpt-5.4-nano" reasoning_effort: high timeout_seconds: 240 -api_key_env: OPENROUTER_API_KEY service_tier: flex diff --git a/internal/profile/builtin/repository_test.go b/internal/profile/builtin/repository_test.go index 9c825b9..2cbf578 100644 --- a/internal/profile/builtin/repository_test.go +++ b/internal/profile/builtin/repository_test.go @@ -7,6 +7,7 @@ import ( "strings" "testing" + "gitea.maximumdirect.net/eric/promptkit/internal/backend" "gitea.maximumdirect.net/eric/promptkit/internal/domain" "gitea.maximumdirect.net/eric/promptkit/internal/profile" "gopkg.in/yaml.v3" @@ -28,6 +29,12 @@ func TestBuiltInProfilesValidateThroughRepository(t *testing.T) { if p.ID != id { t.Fatalf("expected profile id %q, got %q", id, p.ID) } + if p.BackendID != backend.OpenRouterID { + t.Fatalf("expected profile %q to select %q, got %q", id, backend.OpenRouterID, p.BackendID) + } + if p.Endpoint != "" || p.APIKeyEnv != "" { + t.Fatalf("expected profile %q to inherit backend connection settings, got endpoint=%q api_key_env=%q", id, p.Endpoint, p.APIKeyEnv) + } }) } } @@ -60,6 +67,15 @@ func loadBuiltInProfileIDs(t *testing.T) map[string]string { if _, ok := raw["api_key"]; ok { t.Fatalf("built-in profile %s contains raw api_key", name) } + if raw["backend"] != backend.OpenRouterID { + t.Fatalf("built-in profile %s does not select %q", name, backend.OpenRouterID) + } + if _, ok := raw["endpoint"]; ok { + t.Fatalf("built-in profile %s repeats endpoint", name) + } + if _, ok := raw["api_key_env"]; ok { + t.Fatalf("built-in profile %s repeats api_key_env", name) + } id, ok := raw["id"].(string) if !ok || strings.TrimSpace(id) == "" { t.Fatalf("built-in profile %s has missing id", name) diff --git a/internal/profile/filesystem_repository.go b/internal/profile/filesystem_repository.go index ad3a093..a3f731c 100644 --- a/internal/profile/filesystem_repository.go +++ b/internal/profile/filesystem_repository.go @@ -121,6 +121,7 @@ func loadProfile(ctx context.Context, fsys fs.FS, root string, id string) (*doma if prof.ID != id { continue } + prof.BackendID = strings.TrimSpace(prof.BackendID) if err := validateProfile(&prof); err != nil { if errors.Is(err, ErrRawAPIKeyNotAllowed) { return nil, fmt.Errorf("%w: %s", err, relPath) @@ -189,8 +190,8 @@ func validateProfile(p *domain.ExecutionProfile) error { if strings.TrimSpace(p.ID) == "" { return errors.New("id is required") } - if strings.TrimSpace(p.Endpoint) == "" { - return errors.New("endpoint is required") + if strings.TrimSpace(p.BackendID) == "" && strings.TrimSpace(p.Endpoint) == "" { + return errors.New("backend or endpoint is required") } if strings.TrimSpace(p.Model) == "" { return errors.New("model is required") diff --git a/internal/profile/repository_test.go b/internal/profile/repository_test.go index bb36151..97b70a1 100644 --- a/internal/profile/repository_test.go +++ b/internal/profile/repository_test.go @@ -52,6 +52,43 @@ func TestFilesystemRepository_GetProfile(t *testing.T) { } }) + t.Run("backend and endpoint connection matrix", func(t *testing.T) { + tests := []struct { + name string + connection string + wantBackend string + wantEndpoint string + wantErr bool + }{ + {name: "backend only", connection: "backend: ' openrouter '", wantBackend: "openrouter"}, + {name: "endpoint only", connection: "endpoint: http://localhost:8000/v1", wantEndpoint: "http://localhost:8000/v1"}, + {name: "both", connection: "backend: openrouter\nendpoint: http://localhost:8000/v1", wantBackend: "openrouter", wantEndpoint: "http://localhost:8000/v1"}, + {name: "neither", wantErr: true}, + {name: "blank backend", connection: "backend: ' '", wantErr: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + id := "connection-" + strings.ReplaceAll(tt.name, " ", "-") + writeProfileTestFile(t, filepath.Join(tmpDir, id+".yaml"), "id: "+id+"\nmodel: model\n"+tt.connection+"\n") + + p, err := repo.GetProfile(ctx, id) + if tt.wantErr { + if !errors.Is(err, ErrInvalidProfile) { + t.Fatalf("expected ErrInvalidProfile, got %v", err) + } + return + } + if err != nil { + t.Fatalf("expected profile to load, got %v", err) + } + if p.BackendID != tt.wantBackend || p.Endpoint != tt.wantEndpoint { + t.Fatalf("unexpected connection values: backend=%q endpoint=%q", p.BackendID, p.Endpoint) + } + }) + } + }) + t.Run("valid profile with api_key_env", func(t *testing.T) { p, err := repo.GetProfile(ctx, "local-secure") if err != nil { diff --git a/internal/usecase/runner.go b/internal/usecase/runner.go index 4064161..89f559f 100644 --- a/internal/usecase/runner.go +++ b/internal/usecase/runner.go @@ -40,6 +40,7 @@ var ( type Runner struct { promptDefs promptdef.Repository profiles profile.Repository + backends BackendResolver artifacts artifact.Reader renderer prompt.Renderer llm llm.Client @@ -47,20 +48,27 @@ type Runner struct { repairer OutputRepairer } +// BackendResolver resolves one normalized backend ID. +type BackendResolver interface { + GetBackend(string) (domain.Backend, error) +} + func NewRunner( promptDefs promptdef.Repository, profiles profile.Repository, + backends BackendResolver, artifacts artifact.Reader, renderer prompt.Renderer, llmClient llm.Client, validator validate.Validator, ) *Runner { - return NewRunnerWithRepairer(promptDefs, profiles, artifacts, renderer, llmClient, validator, nil) + return NewRunnerWithRepairer(promptDefs, profiles, backends, artifacts, renderer, llmClient, validator, nil) } func NewRunnerWithRepairer( promptDefs promptdef.Repository, profiles profile.Repository, + backends BackendResolver, artifacts artifact.Reader, renderer prompt.Renderer, llmClient llm.Client, @@ -70,6 +78,7 @@ func NewRunnerWithRepairer( return &Runner{ promptDefs: promptDefs, profiles: profiles, + backends: backends, artifacts: artifacts, renderer: renderer, llm: llmClient, @@ -153,6 +162,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes PromptHash: prepared.PromptHash, RenderedPromptHash: prepared.RenderedPromptHash, SelectedProfileID: prepared.SelectedProfileID, + SelectedBackendID: prepared.SelectedBackendID, ModelName: prepared.EffectiveModelParams.Model, Endpoint: prepared.EffectiveModelParams.Endpoint, EffectiveModelParams: prepared.EffectiveModelParams, @@ -193,7 +203,20 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err) } - effectiveModel, targetPresence, err := resolveExecutionTarget(execProfile, req.Execution) + var selectedBackend *domain.Backend + if backendID := strings.TrimSpace(execProfile.BackendID); backendID != "" { + execProfile.BackendID = backendID + if r.backends == nil { + return nil, fmt.Errorf("%w: backend %q cannot be resolved", ErrProfileLoad, backendID) + } + resolvedBackend, resolveErr := r.backends.GetBackend(backendID) + if resolveErr != nil { + return nil, fmt.Errorf("%w: backend %q: %w", ErrProfileLoad, backendID, resolveErr) + } + selectedBackend = &resolvedBackend + } + + effectiveModel, targetPresence, err := resolveExecutionTarget(selectedBackend, execProfile, req.Execution) if err != nil { return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err) } @@ -239,6 +262,7 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr PromptVersion: def.Version, PromptHash: promptDefinitionHash, SelectedProfileID: selectedProfileID, + SelectedBackendID: effectiveModel.BackendID, EffectiveModelParams: effectiveModel, TargetPresence: targetPresence, OutputContract: effectiveContract, @@ -338,6 +362,9 @@ func (r *Runner) shouldAttemptRepair(contract domain.OutputContract, validationR func mergeExecutionTarget(base domain.ExecutionTarget, override domain.ExecutionTarget) domain.ExecutionTarget { out := base + if strings.TrimSpace(override.BackendID) != "" { + out.BackendID = override.BackendID + } if override.Endpoint != "" { out.Endpoint = override.Endpoint } @@ -367,6 +394,7 @@ func mergeExecutionTarget(base domain.ExecutionTarget, override domain.Execution } if override.APIKeyRequired { out.APIKeyRequired = true + out.APIKeyEnv = "" } if len(override.ExtraParams) > 0 { out.ExtraParams = copyExtraParams(override.ExtraParams) @@ -426,8 +454,9 @@ func mergeExecutionTargetOverride(base domain.ExecutionTarget, override domain.E return out, presence, nil } -func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) { +func resolveExecutionTarget(backendValue *domain.Backend, profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) { out := defaults.ExecutionTargetDefault() + out = mergeExecutionTarget(out, backendToTarget(backendValue)) out = mergeExecutionTarget(out, executionProfileToTarget(profileValue)) var presence domain.ExecutionTargetPresence if override != nil { @@ -461,8 +490,13 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget if p == nil { return domain.ExecutionTarget{} } + endpoint := p.Endpoint + if strings.TrimSpace(endpoint) == "" { + endpoint = "" + } return domain.ExecutionTarget{ - Endpoint: p.Endpoint, + BackendID: p.BackendID, + Endpoint: endpoint, Model: p.Model, Temperature: p.Temperature, MaxTokens: p.MaxTokens, @@ -476,6 +510,18 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget } } +func backendToTarget(value *domain.Backend) domain.ExecutionTarget { + if value == nil { + return domain.ExecutionTarget{} + } + return domain.ExecutionTarget{ + BackendID: value.ID, + Endpoint: value.Endpoint, + APIKeyEnv: value.APIKeyEnv, + ExtraParams: copyExtraParams(value.ExtraParams), + } +} + func copyExtraParams(src map[string]any) map[string]any { if len(src) == 0 { return nil diff --git a/internal/usecase/runner_test.go b/internal/usecase/runner_test.go index d0b7ae6..d8c63d2 100644 --- a/internal/usecase/runner_test.go +++ b/internal/usecase/runner_test.go @@ -34,6 +34,18 @@ type fakeExecutionProfileRepo struct { lastID string } +type fakeBackendResolver struct { + backends map[string]domain.Backend +} + +func (f fakeBackendResolver) GetBackend(id string) (domain.Backend, error) { + value, ok := f.backends[id] + if !ok { + return domain.Backend{}, errors.New("backend not found") + } + return value, nil +} + func (f *fakeExecutionProfileRepo) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) { f.lastID = id if f.err != nil { @@ -167,7 +179,7 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) { renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}} llmClient := &fakeLLM{forbid: true} - runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil) + runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil) prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ PromptID: "p", PromptVersion: "1", @@ -257,7 +269,7 @@ func TestRunnerPrepareSelectedProfileDoesNotExistFails(t *testing.T) { } func TestRunnerPreparePromptLoadFailure(t *testing.T) { - runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil) + runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil) _, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p"}) if !errors.Is(err, ErrPromptLoad) { t.Fatalf("expected ErrPromptLoad, got %v", err) @@ -281,7 +293,7 @@ func TestRunnerPrepareRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) { ServiceTier: "priority", }, }} - runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil) + runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil) prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ PromptID: "p", @@ -379,12 +391,12 @@ func TestRunnerPrepareRequestNumericOverridePresence(t *testing.T) { TopP: 0.8, TimeoutSeconds: 45, }, - }}, + }}, nil, + defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, - nil, - ) + nil) prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ PromptID: "p", @@ -426,12 +438,12 @@ func TestRunnerPrepareInvalidRequestNumericOverridesFail(t *testing.T) { t.Run(tc.name, func(t *testing.T) { runner := NewRunner( &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, + defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, - nil, - ) + nil) _, err := runner.Prepare(context.Background(), domain.RunRequest{ PromptID: "p", @@ -458,7 +470,7 @@ func TestRunnerPrepareSelectedProfileBeatsBuiltInDefault(t *testing.T) { ServiceTier: "priority", }, }} - runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil) + runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil) prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ PromptID: "p", @@ -495,12 +507,12 @@ func TestRunnerPrepareFileBackedPromptBodiesRenderCorrectly(t *testing.T) { llmClient := &fakeLLM{forbid: true} runner := NewRunner( promptdef.NewFilesystemRepository(promptDir), - profile.NewFilesystemRepository(profileDir), + profile.NewFilesystemRepository(profileDir), nil, + reader, prompt.NewGoRenderer(), llmClient, - nil, - ) + nil) prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ PromptID: "valid-file-backed", @@ -526,12 +538,13 @@ func TestRunnerPrepareRequiredInputMissingFails(t *testing.T) { runner := NewRunner( &fakePromptRepo{def: def}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, + defaultArtifactReader(), prompt.NewGoRenderer(), &fakeLLM{forbid: true}, - nil, - ) + nil) + _, err := runner.Prepare(context.Background(), domain.RunRequest{ PromptID: "p", ProfileID: "exec", @@ -552,12 +565,13 @@ func TestRunnerPrepareUnknownTemplateInputReferenceFails(t *testing.T) { runner := NewRunner( &fakePromptRepo{def: def}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, + defaultArtifactReader(), prompt.NewGoRenderer(), &fakeLLM{forbid: true}, - nil, - ) + nil) + _, err := runner.Prepare(context.Background(), domain.RunRequest{ PromptID: "p", ProfileID: "exec", @@ -579,7 +593,7 @@ func TestRunnerPrepareAPIKeyEnvNameIncludedButNotResolvedValue(t *testing.T) { execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName}, }} - runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil) + runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil) prepared, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()}) if err != nil { @@ -607,12 +621,12 @@ func TestRunnerPrepareJSONSchemaBuildsStructuredOutputSpec(t *testing.T) { } runner := NewRunner( &fakePromptRepo{def: def}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, + defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, - validator, - ) + validator) prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ PromptID: "p", @@ -651,12 +665,12 @@ func TestRunnerPrepareJSONSchemaSchemaLoadFailureReturnsValidationError(t *testi validator := &fakeValidator{schemaErr: errors.New("schema unavailable")} runner := NewRunner( &fakePromptRepo{def: def}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, + defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, - validator, - ) + validator) _, err := runner.Prepare(context.Background(), domain.RunRequest{ PromptID: "p", @@ -681,12 +695,12 @@ func TestRunnerRunJSONSchemaSchemaLoadFailureFailsBeforeLLM(t *testing.T) { validator := &fakeValidator{schemaErr: errors.New("schema unavailable")} runner := NewRunner( &fakePromptRepo{def: def}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, + defaultArtifactReader(), defaultRenderer(), llmClient, - validator, - ) + validator) _, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", @@ -836,7 +850,7 @@ func TestRunnerRunSuccessful(t *testing.T) { renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}} - runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil) + runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil) res, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", PromptVersion: "1", @@ -915,7 +929,7 @@ func TestRunnerRunPassesExtraParamsToGenerateRequestTarget(t *testing.T) { }, }} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}} - runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil) + runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil) res, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", @@ -941,7 +955,7 @@ func TestRunnerRunAndPrepareResolveSameProfileAndEffectiveSettings(t *testing.T) }} renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap"}} - runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil) + runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil) req := domain.RunRequest{ PromptID: "p", @@ -1071,7 +1085,7 @@ func TestRunnerRunExplicitRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) }, }} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}} - runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil) + runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil) res, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", @@ -1115,7 +1129,7 @@ func TestRunnerRunSelectedProfileBeatsBuiltInDefault(t *testing.T) { }, }} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}} - runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil) + runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil) res, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", @@ -1142,7 +1156,7 @@ func TestRunnerRunBuiltInDefaultsUsedWhenProfileOmitsOptionalFields(t *testing.T "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"}, }} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}} - runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil) + runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil) res, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", @@ -1172,7 +1186,7 @@ func TestRunnerRunAPIKeyEnvResolvesFromEnvironment(t *testing.T) { execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_TEST_API_KEY"}, }} - runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil) + runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil) res, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()}) if err != nil { @@ -1188,7 +1202,7 @@ func TestRunnerRunAPIKeyEnvMissingEnvironmentValueFailsClearly(t *testing.T) { execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_MISSING_KEY"}, }} - runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil) + runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil) _, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()}) if !errors.Is(err, ErrInvalidRequest) { @@ -1209,7 +1223,7 @@ func TestRunnerRunDirectAPIKeyBypassesMissingEnvAndReachesLLM(t *testing.T) { "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_MISSING_KEY"}, }} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}} - runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil) + runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil) _, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", @@ -1233,7 +1247,7 @@ func TestRunnerPrepareAPIKeyRequiredFailsWithoutDirectKey(t *testing.T) { execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true}, }} - runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil) + runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil) _, err := runner.Prepare(context.Background(), domain.RunRequest{ PromptID: "p", @@ -1252,7 +1266,7 @@ func TestRunnerRunAPIKeyRequiredSucceedsWithDirectKey(t *testing.T) { "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true}, }} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}} - runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil) + runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil) _, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", @@ -1279,7 +1293,7 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideWorks(t *testing.T) { execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"}, }} - runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil) + runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil) res, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", @@ -1304,7 +1318,7 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideBeatsProfile(t *testing.T) { execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: profileEnv}, }} - runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil) + runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil) res, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", @@ -1328,7 +1342,7 @@ func TestRunnerRunAPIKeyValueNotPresentInMetadata(t *testing.T) { execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName}, }} - runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil) + runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil) res, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()}) if err != nil { @@ -1344,7 +1358,7 @@ func TestRunnerRunAPIKeyValueNotPresentInMetadata(t *testing.T) { } func TestRunnerRunPromptLoadFailure(t *testing.T) { - runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil) + runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil) _, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"}) if !errors.Is(err, ErrPromptLoad) { t.Fatalf("expected ErrPromptLoad, got %v", err) @@ -1357,12 +1371,12 @@ func TestRunnerRunPromptLoadFailure(t *testing.T) { func TestRunnerRunArtifactLoadFailure(t *testing.T) { runner := NewRunner( &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, + &fakeArtifactReader{errByURI: map[string]error{"a://bad": errors.New("read failed")}}, &fakeRenderer{rendered: &domain.RenderedPrompt{}}, &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, - nil, - ) + nil) _, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", @@ -1377,12 +1391,13 @@ func TestRunnerRunArtifactLoadFailure(t *testing.T) { func TestRunnerRunPromptRenderFailure(t *testing.T) { runner := NewRunner( &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, + defaultArtifactReader(), &fakeRenderer{err: errors.New("render failed")}, &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, - nil, - ) + nil) + _, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", ProfileID: "exec", @@ -1396,12 +1411,13 @@ func TestRunnerRunPromptRenderFailure(t *testing.T) { func TestRunnerRunLLMFailure(t *testing.T) { runner := NewRunner( &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, + defaultArtifactReader(), defaultRenderer(), &fakeLLM{err: errors.New("llm failed")}, - nil, - ) + nil) + _, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", ProfileID: "exec", @@ -1415,12 +1431,13 @@ func TestRunnerRunLLMFailure(t *testing.T) { func TestRunnerRunCancellationPreservesGenerationCategory(t *testing.T) { runner := NewRunner( &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, + defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ignored"}}, - nil, - ) + nil) + ctx, cancel := context.WithCancel(context.Background()) cancel() @@ -1440,12 +1457,13 @@ func TestRunnerRunCancellationPreservesGenerationCategory(t *testing.T) { func TestRunnerRunLLMInvalidRequestMapsToUsecaseInvalidRequest(t *testing.T) { runner := NewRunner( &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, + defaultArtifactReader(), defaultRenderer(), &fakeLLM{err: llm.ErrInvalidRequest}, - nil, - ) + nil) + _, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", ProfileID: "exec", @@ -1463,12 +1481,13 @@ func TestRunnerRunValidationStillWorks(t *testing.T) { validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}} runner := NewRunner( &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, + defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}}, - validator, - ) + validator) + res, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", ProfileID: "exec", @@ -1489,14 +1508,17 @@ func TestRunnerRunStructuredRepairRemainsBoundedAndUsesEffectiveModelSettings(t runner := NewRunnerWithRepairer( &fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ - "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", TimeoutSeconds: 55}, + "exec": {ID: "exec", BackendID: "custom", Model: "profile-model", TimeoutSeconds: 55}, + }}, fakeBackendResolver{backends: map[string]domain.Backend{ + "custom": {ID: "custom", Endpoint: "http://backend/v1"}, }}, + defaultArtifactReader(), defaultRenderer(), llmClient, validate.NewStandardValidator("."), - repairer, - ) + repairer) + res, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", ProfileID: "exec", @@ -1518,6 +1540,12 @@ func TestRunnerRunStructuredRepairRemainsBoundedAndUsesEffectiveModelSettings(t if repairer.reqs[0].Target.TimeoutSeconds != 22 { t.Fatalf("expected repair to use effective timeout, got %d", repairer.reqs[0].Target.TimeoutSeconds) } + if llmClient.lastReq.Target.BackendID != "custom" || + repairer.reqs[0].Target.BackendID != "custom" || + res.SelectedBackendID != "custom" { + t.Fatalf("expected backend identity in generation, repair, and result: generate=%q repair=%q result=%q", + llmClient.lastReq.Target.BackendID, repairer.reqs[0].Target.BackendID, res.SelectedBackendID) + } } func TestRunnerRunJSONSchemaRepairCarriesStructuredOutputSpec(t *testing.T) { @@ -1546,13 +1574,13 @@ func TestRunnerRunJSONSchemaRepairCarriesStructuredOutputSpec(t *testing.T) { llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: `{"events":[1]}`}} runner := NewRunnerWithRepairer( &fakePromptRepo{def: def}, - &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, + &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, + defaultArtifactReader(), defaultRenderer(), llmClient, validator, - repairer, - ) + repairer) _, err := runner.Run(context.Background(), domain.RunRequest{ PromptID: "p", @@ -1629,13 +1657,12 @@ func TestResolveExecutionTargetProfileValuesPopulateAllSupportedFields(t *testin ServiceTier: "priority", ReasoningEffort: "low", APIKeyEnv: "PROFILE_KEY", - APIKeyRequired: true, ExtraParams: map[string]any{ "profile_option": "enabled", }, } - target, presence, err := resolveExecutionTarget(profileValue, nil) + target, presence, err := resolveExecutionTarget(nil, profileValue, nil) if err != nil { t.Fatalf("expected no error, got %v", err) } @@ -1690,7 +1717,7 @@ func TestResolveExecutionTargetRuntimeOverridesBeatProfileForAllOverrideableFiel }, } - target, presence, err := resolveExecutionTarget(profileValue, override) + target, presence, err := resolveExecutionTarget(nil, profileValue, override) if err != nil { t.Fatalf("expected no error, got %v", err) } @@ -1813,6 +1840,118 @@ func defaultExecutionProfile() *domain.ExecutionProfile { } } +func TestResolveExecutionTargetUsesBackendProfileAndRequestPrecedence(t *testing.T) { + backendValue := &domain.Backend{ + ID: "custom", + Endpoint: "http://backend/v1", + APIKeyEnv: "BACKEND_KEY", + ExtraParams: map[string]any{"backend": true}, + } + profileValue := &domain.ExecutionProfile{ + ID: "exec", + BackendID: "custom", + Endpoint: "http://profile/v1", + Model: "profile-model", + APIKeyEnv: "PROFILE_KEY", + ExtraParams: map[string]any{"profile": true}, + } + override := &domain.ExecutionTargetOverride{ + Endpoint: "http://request/v1", + APIKeyEnv: "REQUEST_KEY", + ExtraParams: map[string]any{"request": true}, + } + + target, _, err := resolveExecutionTarget(backendValue, profileValue, override) + if err != nil { + t.Fatalf("resolve target: %v", err) + } + if target.BackendID != "custom" { + t.Fatalf("endpoint override changed backend identity: %+v", target) + } + if target.Endpoint != "http://request/v1" || target.APIKeyEnv != "REQUEST_KEY" { + t.Fatalf("request values did not win: %+v", target) + } + if !reflect.DeepEqual(target.ExtraParams, map[string]any{"request": true}) { + t.Fatalf("expected whole-map request replacement, got %#v", target.ExtraParams) + } + + target, _, err = resolveExecutionTarget(backendValue, &domain.ExecutionProfile{ + ID: "exec", BackendID: "custom", Model: "profile-model", + }, nil) + if err != nil { + t.Fatalf("resolve backend defaults: %v", err) + } + if target.Endpoint != backendValue.Endpoint || + target.APIKeyEnv != backendValue.APIKeyEnv || + !reflect.DeepEqual(target.ExtraParams, backendValue.ExtraParams) { + t.Fatalf("backend defaults were not inherited: %+v", target) + } +} + +func TestRunnerPrepareBackendResolutionAndCredentialPrecedence(t *testing.T) { + resolver := fakeBackendResolver{backends: map[string]domain.Backend{ + "custom": {ID: "custom", Endpoint: "http://backend/v1", APIKeyEnv: "BACKEND_KEY"}, + }} + promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)} + + t.Run("unknown backend is a profile load failure", func(t *testing.T) { + runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ + "exec": {ID: "exec", BackendID: "unknown", Model: "model"}, + }}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil) + _, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()}) + if !errors.Is(err, ErrProfileLoad) { + t.Fatalf("expected ErrProfileLoad, got %v", err) + } + }) + + t.Run("nil resolver is a profile load failure", func(t *testing.T) { + runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ + "exec": {ID: "exec", BackendID: "custom", Model: "model"}, + }}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil) + _, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()}) + if !errors.Is(err, ErrProfileLoad) { + t.Fatalf("expected ErrProfileLoad, got %v", err) + } + }) + + t.Run("request environment wins", func(t *testing.T) { + t.Setenv("REQUEST_KEY", "request-secret") + runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ + "exec": {ID: "exec", BackendID: "custom", Model: "model", APIKeyEnv: "PROFILE_KEY"}, + }}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil) + prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ + PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(), + Execution: &domain.ExecutionTargetOverride{APIKeyEnv: "REQUEST_KEY"}, + }) + if err != nil { + t.Fatalf("prepare: %v", err) + } + if prepared.EffectiveModelParams.APIKeyEnv != "REQUEST_KEY" { + t.Fatalf("unexpected credential source: %+v", prepared.EffectiveModelParams) + } + }) + + t.Run("required direct key clears inherited environment", func(t *testing.T) { + t.Setenv("BACKEND_KEY", "backend-secret") + runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ + "exec": {ID: "exec", BackendID: "custom", Model: "model", APIKeyRequired: true}, + }}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil) + _, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()}) + if !errors.Is(err, ErrAPIKeyRequired) { + t.Fatalf("expected ErrAPIKeyRequired, got %v", err) + } + prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ + PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(), APIKey: "direct-secret", + }) + if err != nil { + t.Fatalf("prepare with direct key: %v", err) + } + if prepared.EffectiveModelParams.APIKeyEnv != "" { + t.Fatalf("expected inherited environment name to be cleared, got %q", prepared.EffectiveModelParams.APIKeyEnv) + } + }) +} + func defaultArtifactReader() *fakeArtifactReader { return &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{ "a://ok": {Body: []byte("x"), Hash: hashString("x")}, @@ -1838,10 +1977,11 @@ func intPtr(v int) *int { func newMinimalRunner(promptRepo *fakePromptRepo, execRepo *fakeExecutionProfileRepo) *Runner { return NewRunner( promptRepo, - execRepo, + execRepo, nil, + defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, - nil, - ) + nil) + } diff --git a/json.go b/json.go index 3a6e3cb..14673ce 100644 --- a/json.go +++ b/json.go @@ -26,6 +26,7 @@ func (r PreparedRun) MarshalJSON() ([]byte, error) { PromptVersion string `json:"prompt_version,omitempty"` PromptHash string `json:"prompt_hash,omitempty"` SelectedProfileID string `json:"selected_profile_id"` + SelectedBackendID string `json:"selected_backend_id,omitempty"` EffectiveModelParams ExecutionTarget `json:"effective_model_params"` OutputContract OutputContract `json:"output_contract"` StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"` @@ -41,6 +42,7 @@ func (r PreparedRun) MarshalJSON() ([]byte, error) { PromptVersion: r.PromptVersion, PromptHash: r.PromptHash, SelectedProfileID: r.SelectedProfileID, + SelectedBackendID: r.SelectedBackendID, EffectiveModelParams: r.EffectiveModelParams, OutputContract: r.OutputContract, StructuredOutput: r.StructuredOutput, @@ -81,6 +83,7 @@ func (r RunResult) MarshalJSON() ([]byte, error) { PromptHash: r.PromptHash, RenderedPromptHash: r.RenderedPromptHash, SelectedProfileID: r.SelectedProfileID, + SelectedBackendID: r.SelectedBackendID, ModelName: r.ModelName, Endpoint: r.Endpoint, EffectiveModelParams: r.EffectiveModelParams, @@ -110,6 +113,7 @@ func (r *RunResult) UnmarshalJSON(data []byte) error { PromptHash: wire.PromptHash, RenderedPromptHash: wire.RenderedPromptHash, SelectedProfileID: wire.SelectedProfileID, + SelectedBackendID: wire.SelectedBackendID, ModelName: wire.ModelName, Endpoint: wire.Endpoint, EffectiveModelParams: wire.EffectiveModelParams, @@ -136,6 +140,7 @@ type runResultJSON struct { PromptHash string `json:"prompt_hash,omitempty"` RenderedPromptHash string `json:"rendered_prompt_hash"` SelectedProfileID string `json:"selected_profile_id"` + SelectedBackendID string `json:"selected_backend_id,omitempty"` ModelName string `json:"model_name"` Endpoint string `json:"endpoint"` EffectiveModelParams ExecutionTarget `json:"effective_model_params"` diff --git a/profiles.go b/profiles.go index 6fbb2f1..8d22f7b 100644 --- a/profiles.go +++ b/profiles.go @@ -6,16 +6,21 @@ import ( "fmt" "strings" + "gitea.maximumdirect.net/eric/promptkit/internal/backend" "gitea.maximumdirect.net/eric/promptkit/internal/domain" "gitea.maximumdirect.net/eric/promptkit/internal/profile" ) +// BackendOpenRouter identifies Promptkit's built-in OpenRouter backend. +const BackendOpenRouter = backend.OpenRouterID + // OpenAICompatibleProfile returns an ordinary in-memory Profile for an // OpenAI-compatible chat-completions endpoint. // // It does not register global state, maintain a model catalog, or resolve // credentials. If APIKeyRequired is true, callers satisfy it with -// RunRequest.APIKey. Raw API keys do not belong in profiles. +// RunRequest.APIKey or an explicit request ExecutionTargetOverride.APIKeyEnv. +// Raw API keys do not belong in profiles. // // The function copies the ExtraParams map itself but does not recursively copy // nested values. Validation and a deep copy occur when NewEngine applies a @@ -23,6 +28,7 @@ import ( func OpenAICompatibleProfile(cfg OpenAICompatibleProfileConfig) Profile { return Profile{ ID: cfg.ID, + BackendID: cfg.BackendID, Endpoint: cfg.Endpoint, Model: cfg.Model, Temperature: cfg.Temperature, @@ -85,6 +91,7 @@ func toDomainProfile(publicProfile Profile) (domain.ExecutionProfile, error) { } prof := domain.ExecutionProfile{ ID: strings.TrimSpace(publicProfile.ID), + BackendID: strings.TrimSpace(publicProfile.BackendID), Endpoint: publicProfile.Endpoint, Model: publicProfile.Model, Temperature: publicProfile.Temperature, @@ -106,8 +113,8 @@ func validatePublicProfile(prof domain.ExecutionProfile) error { if strings.TrimSpace(prof.ID) == "" { return errors.New("id is required") } - if strings.TrimSpace(prof.Endpoint) == "" { - return errors.New("endpoint is required") + if strings.TrimSpace(prof.BackendID) == "" && strings.TrimSpace(prof.Endpoint) == "" { + return errors.New("backend or endpoint is required") } if strings.TrimSpace(prof.Model) == "" { return errors.New("model is required") diff --git a/public_contract_test.go b/public_contract_test.go index e08d55f..bcf0465 100644 --- a/public_contract_test.go +++ b/public_contract_test.go @@ -3,6 +3,7 @@ package promptkit_test import ( "context" "encoding/json" + "errors" "fmt" "strings" "sync" @@ -26,6 +27,62 @@ func TestPreparedRunJSONOmitsZeroTimingValues(t *testing.T) { } } +func TestBackendIdentityJSONNamesAndOmission(t *testing.T) { + values := []struct { + name string + value any + field string + }{ + {name: "execution target", value: promptkit.ExecutionTarget{BackendID: promptkit.BackendOpenRouter}, field: "backend_id"}, + {name: "prepared run", value: promptkit.PreparedRun{SelectedBackendID: promptkit.BackendOpenRouter}, field: "selected_backend_id"}, + {name: "run result", value: promptkit.RunResult{SelectedBackendID: promptkit.BackendOpenRouter}, field: "selected_backend_id"}, + } + for _, tt := range values { + t.Run(tt.name, func(t *testing.T) { + payload, err := json.Marshal(tt.value) + if err != nil { + t.Fatalf("marshal populated value: %v", err) + } + var object map[string]any + if err := json.Unmarshal(payload, &object); err != nil { + t.Fatalf("decode populated value: %v", err) + } + if object[tt.field] != promptkit.BackendOpenRouter { + t.Fatalf("expected %s=%q, got %s", tt.field, promptkit.BackendOpenRouter, payload) + } + }) + } + + emptyValues := []any{promptkit.ExecutionTarget{}, promptkit.PreparedRun{}, promptkit.RunResult{}} + for _, value := range emptyValues { + payload, err := json.Marshal(value) + if err != nil { + t.Fatalf("marshal empty value: %v", err) + } + if strings.Contains(string(payload), `"backend_id"`) || strings.Contains(string(payload), `"selected_backend_id"`) { + t.Fatalf("empty backend identity was not omitted: %s", payload) + } + } +} + +func TestUnknownProfileBackendHasProfileLoadIdentity(t *testing.T) { + engine, err := promptkit.NewEngine(promptkit.Config{}, + promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."), + promptkit.WithProfiles(promptkit.Profile{ID: "profile", BackendID: "unknown", Model: "model"}), + ) + if err != nil { + t.Fatalf("construct engine: %v", err) + } + + _, err = engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"}) + if !errors.Is(err, promptkit.ErrProfileLoad) { + t.Fatalf("expected ErrProfileLoad, got %v", err) + } + if errors.Is(err, promptkit.ErrInvalidRequest) { + t.Fatalf("unknown backend should not have invalid-request identity: %v", err) + } +} + func TestPreparedRunJSONTimingRoundTrips(t *testing.T) { start := time.Date(2026, time.July, 29, 12, 0, 0, 0, time.UTC) prepared := promptkit.PreparedRun{ diff --git a/types.go b/types.go index f190ede..b44e63a 100644 --- a/types.go +++ b/types.go @@ -126,8 +126,12 @@ type PreparedRun struct { // SelectedProfileID is the explicit request profile or prompt default that // supplied execution settings. SelectedProfileID string `json:"selected_profile_id"` + // SelectedBackendID is the selected profile's normalized backend ID. It is + // empty for an endpoint-only profile. + SelectedBackendID string `json:"selected_backend_id,omitempty"` // EffectiveModelParams contains framework defaults overlaid by the selected - // profile and then request overrides. It excludes resolved API-key values. + // backend, profile, and then request overrides. It excludes resolved API-key + // values. EffectiveModelParams ExecutionTarget `json:"effective_model_params"` // OutputContract is the complete effective output contract. OutputContract OutputContract `json:"output_contract"` @@ -180,6 +184,9 @@ type RunResult struct { RenderedPromptHash string `json:"rendered_prompt_hash"` // SelectedProfileID identifies the profile used for execution. SelectedProfileID string `json:"selected_profile_id"` + // SelectedBackendID is the selected profile's normalized backend ID. It is + // empty for an endpoint-only profile. + SelectedBackendID string `json:"selected_backend_id,omitempty"` // ModelName is the effective model name and equals // EffectiveModelParams.Model. ModelName string `json:"model_name"` @@ -259,6 +266,10 @@ type ArtifactReader interface { // ExecutionTarget represents effective model runtime settings and has a stable // JSON representation. It never exposes a resolved API-key value. type ExecutionTarget struct { + // BackendID is the effective routing identity selected by the profile. It + // remains unchanged when a profile or request overrides Endpoint and is + // empty for endpoint-only profiles. + BackendID string `json:"backend_id,omitempty"` // Endpoint is the model-provider base URL. Endpoint string `json:"endpoint"` // Model is the provider model identifier. @@ -323,9 +334,10 @@ type ExecutionTargetOverride struct { // Profile is an in-memory execution profile for library consumers. // // It is equivalent to a loaded profile file after validation. Raw API keys do -// not belong in profiles; use APIKeyRequired to require callers to provide -// RunRequest.APIKey for each request, or use profile YAML api_key_env with file -// and FS profile sources. Profile has no stable JSON representation. +// not belong in profiles; use APIKeyRequired to require callers to provide a +// RunRequest.APIKey or explicit request ExecutionTargetOverride.APIKeyEnv, or +// use profile YAML api_key_env with file and FS profile sources. Profile has no +// stable JSON representation. // // WithProfiles validates and copies Profile values during NewEngine. Numeric // zero, blank strings, and an empty ExtraParams map inherit framework defaults; @@ -333,7 +345,12 @@ type ExecutionTargetOverride struct { type Profile struct { // ID is the required non-blank profile identifier. WithProfiles trims it. ID string - // Endpoint is the required non-blank model-provider base URL. + // BackendID optionally selects an engine backend. WithProfiles trims it. + // Backend membership is checked when a request selects the profile. + BackendID string + // Endpoint is the model-provider base URL. It is required only when + // BackendID is blank and otherwise overrides the backend endpoint when + // non-blank. Endpoint string // Model is the required non-blank provider model identifier. Model string @@ -351,8 +368,9 @@ type Profile struct { // ReasoningEffort is optional; a blank value inherits the framework // default. ReasoningEffort string - // APIKeyRequired requires a non-blank RunRequest.APIKey. It does not store a - // credential or enable environment lookup. + // APIKeyRequired clears a backend's inherited API-key environment name and + // requires a non-blank RunRequest.APIKey unless the request explicitly + // supplies ExecutionTargetOverride.APIKeyEnv. It does not store a credential. APIKeyRequired bool // ExtraParams contains provider-specific JSON-compatible values. An empty // map inherits framework defaults. WithProfiles validates and deeply copies @@ -364,13 +382,15 @@ type Profile struct { // profile. // // It contains ordinary profile fields for OpenAI-compatible chat-completions -// endpoints. APIKeyRequired is satisfied by RunRequest.APIKey. Raw API keys do -// not belong in this config. OpenAICompatibleProfileConfig has no stable JSON +// endpoints. APIKeyRequired follows Profile.APIKeyRequired. Raw API keys do not +// belong in this config. OpenAICompatibleProfileConfig has no stable JSON // representation and is not validated until its resulting Profile is supplied // through WithProfiles to NewEngine. type OpenAICompatibleProfileConfig struct { // ID becomes Profile.ID. ID string + // BackendID becomes Profile.BackendID. + BackendID string // Endpoint becomes Profile.Endpoint. Endpoint string // Model becomes Profile.Model.