Resolve profiles through built-in backends

This commit is contained in:
2026-07-29 17:10:34 +00:00
parent 8d00354c59
commit 810f80e7c9
41 changed files with 538 additions and 173 deletions

View File

@@ -32,6 +32,7 @@ func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
PromptVersion: prepared.PromptVersion, PromptVersion: prepared.PromptVersion,
PromptHash: prepared.PromptHash, PromptHash: prepared.PromptHash,
SelectedProfileID: prepared.SelectedProfileID, SelectedProfileID: prepared.SelectedProfileID,
SelectedBackendID: prepared.SelectedBackendID,
EffectiveModelParams: fromDomainExecutionTarget(prepared.EffectiveModelParams), EffectiveModelParams: fromDomainExecutionTarget(prepared.EffectiveModelParams),
OutputContract: fromDomainOutputContract(prepared.OutputContract), OutputContract: fromDomainOutputContract(prepared.OutputContract),
StructuredOutput: fromDomainStructuredOutputSpec(prepared.StructuredOutput), StructuredOutput: fromDomainStructuredOutputSpec(prepared.StructuredOutput),
@@ -59,6 +60,7 @@ func fromDomainRunResult(result *domain.RunResult) *RunResult {
PromptHash: result.PromptHash, PromptHash: result.PromptHash,
RenderedPromptHash: result.RenderedPromptHash, RenderedPromptHash: result.RenderedPromptHash,
SelectedProfileID: result.SelectedProfileID, SelectedProfileID: result.SelectedProfileID,
SelectedBackendID: result.SelectedBackendID,
ModelName: result.ModelName, ModelName: result.ModelName,
Endpoint: result.Endpoint, Endpoint: result.Endpoint,
EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams), EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams),
@@ -151,6 +153,7 @@ func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain
func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget { func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
return ExecutionTarget{ return ExecutionTarget{
BackendID: target.BackendID,
Endpoint: target.Endpoint, Endpoint: target.Endpoint,
Model: target.Model, Model: target.Model,
Temperature: target.Temperature, Temperature: target.Temperature,

View File

@@ -136,7 +136,7 @@ A profile supplies model execution settings:
```yaml ```yaml
id: local-summary id: local-summary
endpoint: http://localhost:8000/v1 backend: openrouter
model: example-model model: example-model
temperature: 0.2 temperature: 0.2
max_tokens: 500 max_tokens: 500
@@ -144,7 +144,6 @@ top_p: 0.95
timeout_seconds: 90 timeout_seconds: 90
service_tier: flex service_tier: flex
reasoning_effort: medium reasoning_effort: medium
api_key_env: EXAMPLE_API_KEY
extra_params: extra_params:
provider_option: enabled provider_option: enabled
``` ```
@@ -152,7 +151,8 @@ extra_params:
| Field | Required | Meaning | | Field | Required | Meaning |
| --- | --- | --- | | --- | --- | --- |
| `id` | yes | Non-empty profile identifier. IDs must be unique within one source. | | `id` | yes | Non-empty profile identifier. IDs must be unique within one source. |
| `endpoint` | yes | Non-empty OpenAI-compatible base URL, including an API version path when required. | | `backend` | unless `endpoint` is present | Backend registry ID. It is trimmed and registry membership is checked when the profile is prepared. |
| `endpoint` | unless `backend` is present | Non-empty OpenAI-compatible base URL, including an API version path when required. When both connection fields are present, this overrides the backend endpoint without changing backend identity. |
| `model` | yes | Non-empty provider model name. | | `model` | yes | Non-empty provider model name. |
| `temperature` | no | Number from 0 through 2. | | `temperature` | no | Number from 0 through 2. |
| `max_tokens` | no | Integer zero or greater. | | `max_tokens` | no | Integer zero or greater. |
@@ -166,6 +166,9 @@ extra_params:
Raw `api_key` is prohibited in profile YAML. Store only an environment Raw `api_key` is prohibited in profile YAML. Store only an environment
variable name in `api_key_env`. variable name in `api_key_env`.
Promptkit does not infer a backend from a model or endpoint. Endpoint-only
profiles remain supported and have no effective backend ID.
`extra_params` accepts null, booleans, finite numbers, strings, arrays, and `extra_params` accepts null, booleans, finite numbers, strings, arrays, and
objects with string keys. Keys must be non-empty. With the built-in client, objects with string keys. Keys must be non-empty. With the built-in client,
they also cannot collide with the standard fields listed in the they also cannot collide with the standard fields listed in the
@@ -176,8 +179,9 @@ they also cannot collide with the standard fields listed in the
Execution settings resolve in this order: Execution settings resolve in this order:
1. framework defaults; 1. framework defaults;
2. the selected profile; and 2. the selected backend, when the profile names one;
3. request `ExecutionTargetOverride` values. 3. the selected profile; and
4. request `ExecutionTargetOverride` values.
The framework defaults are: The framework defaults are:
@@ -194,8 +198,10 @@ explicit zero is preserved. In particular, an explicit request
`timeout_seconds` of zero disables the per-generation deadline while leaving `timeout_seconds` of zero disables the per-generation deadline while leaving
the caller context and transport timeout intact. the caller context and transport timeout intact.
Non-empty request strings replace profile strings. A non-empty request Non-empty profile strings replace backend defaults, and non-empty request
`ExtraParams` map replaces the profile map rather than merging keys. strings replace both. Backend identity is retained when either layer overrides
the endpoint. A non-empty `extra_params` map at each layer replaces the entire
lower-precedence map rather than merging keys.
The [outbound integration contract](integrations/openai-compatible-chat.md) The [outbound integration contract](integrations/openai-compatible-chat.md)
defines how the effective settings are serialized. defines how the effective settings are serialized.
@@ -217,9 +223,11 @@ invalid matching profile is an error and does not fall back. In-memory
## Built-In Profile Catalog ## Built-In Profile Catalog
Built-ins use the OpenRouter-compatible endpoint and Every built-in selects the `openrouter` backend. The engine's built-in backend
`OPENROUTER_API_KEY`. A custom or in-memory profile with the same ID takes registry supplies `https://openrouter.ai/api/v1` and the environment-variable
precedence. name `OPENROUTER_API_KEY`, so individual profiles contain only model and
generation settings. A custom or in-memory profile with the same profile ID
takes precedence.
| Provider | ID | Model | | Provider | ID | Model |
| --- | --- | --- | | --- | --- | --- |
@@ -270,6 +278,10 @@ prompt, profile, schema, or example files:
- a request can provide a direct `APIKey` or override `APIKeyEnv`; and - a request can provide a direct `APIKey` or override `APIKeyEnv`; and
- a direct request key takes precedence over environment lookup. - a direct request key takes precedence over environment lookup.
After a direct request key, the credential-source precedence is request
`APIKeyEnv`, profile `api_key_env`, then the backend default. An in-memory
profile with `APIKeyRequired` clears an inherited backend environment name and
requires a direct key unless the request explicitly supplies `APIKeyEnv`.
Promptkit validates required credential availability during preparation. Promptkit validates required credential availability during preparation.
Direct keys are excluded from JSON results and redacted by public string Direct keys are excluded from JSON results and redacted by public string
formatters. Environment-variable names may appear in prepared metadata, but formatters. Environment-variable names may appear in prepared metadata, but

View File

@@ -17,10 +17,11 @@ and override semantics consumed by the runner.
## Collaborators ## Collaborators
`Runner` coordinates narrow internal interfaces for prompt definitions, `Runner` coordinates narrow internal interfaces for prompt definitions,
profiles, artifacts, rendering, model generation, and validation. Schema profiles, backend resolution, artifacts, rendering, model generation, and
documents are loaded through the validator's optional schema-loader interface. validation. The root engine supplies an immutable built-in backend registry.
An output repairer can be injected internally, but the ordinary runner Schema documents are loaded through the validator's optional schema-loader
constructor does not enable one. interface. An output repairer can be injected internally, but the ordinary
runner constructor does not enable one.
Each invocation carries its state in request, prepared-run, and result values. Each invocation carries its state in request, prepared-run, and result values.
The runner has no durable run or session store. The runner has no durable run or session store.
@@ -32,20 +33,25 @@ The runner has no durable run or session store.
1. validate the prompt selection and load the prompt definition; 1. validate the prompt selection and load the prompt definition;
2. hash the loaded definition; 2. hash the loaded definition;
3. select the request profile or the prompt's default profile; 3. select the request profile or the prompt's default profile;
4. resolve application-neutral defaults, profile values, and explicit request 4. resolve the profile's backend ID, when present;
overrides in that order; 5. resolve application-neutral defaults, backend defaults, profile values,
5. validate endpoint, model, numeric overrides, and credential requirements; and explicit request overrides in that order;
6. resolve the output contract and load a structured-output schema when 6. validate endpoint, model, numeric overrides, and credential requirements;
7. resolve the output contract and load a structured-output schema when
required; required;
7. load and hash input artifacts; 8. load and hash input artifacts;
8. render and hash the prompt; and 9. render and hash the prompt; and
9. return the effective settings, source identities, messages, hashes, and 10. return the effective settings, source identities, messages, hashes, and
preparation timing. preparation timing.
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
out-of-range values fail as invalid requests. A direct API key takes out-of-range values fail as invalid requests. Endpoint overrides do not change
precedence over environment lookup for execution; secret values remain the selected backend identity. Non-empty extra-parameter maps replace whole
excluded from serialized metadata. lower-precedence maps. A direct API key takes precedence over environment
lookup; otherwise request, profile, and backend environment-variable names
apply in that order. A profile requiring a direct key clears an inherited
backend environment name unless the request supplies its own name. Secret
values remain excluded from serialized metadata.
## Run Flow ## Run Flow
@@ -60,8 +66,10 @@ target, validation errors, prior output, and structured-output specification.
This capability remains internal and is not a public option. This capability remains internal and is not a public option.
A successful result includes the output artifact and raw output, validation A successful result includes the output artifact and raw output, validation
state, prompt and rendered-prompt hashes, selected profile, effective settings, state, prompt and rendered-prompt hashes, selected profile and backend,
input hashes, token usage, a generated run identifier, and UTC timing. effective settings, input hashes, token usage, a generated run identifier, and
UTC timing. The same effective target, including backend identity, reaches
generation and any repair attempt.
## Failure Categories ## Failure Categories
@@ -71,13 +79,16 @@ validation failures. Wrapping preserves the package identities mapped by the
public facade and retains collaborator identities where they are part of the public facade and retains collaborator identities where they are part of the
internal contract. Context cancellation propagates through the invoked internal contract. Context cancellation propagates through the invoked
collaborator and is classified by the owning operation. collaborator and is classified by the owning operation.
An unknown selected backend, or a selected backend with no configured resolver,
is classified as a profile-load failure.
## Test Ownership And Changes ## Test Ownership And Changes
The [runner tests](../../internal/usecase/runner_test.go) own preparation order, The [runner tests](../../internal/usecase/runner_test.go) own preparation order,
selection and override precedence, schema-before-generation behavior, hashing, selection and override precedence, schema-before-generation behavior, hashing,
generation and validation outcomes, bounded repair, credentials and redaction, generation and validation outcomes, backend propagation, bounded repair,
error categories, artifact metadata, usage, and timing. credentials and redaction, error categories, artifact metadata, usage, and
timing.
Changes to orchestration should continue to use the existing package Changes to orchestration should continue to use the existing package
interfaces, keep request state local to an invocation, and preserve `Run`'s use interfaces, keep request state local to an invocation, and preserve `Run`'s use

View File

@@ -24,13 +24,19 @@ duplicate detection, and source containment:
`internal/profile` loads and validates execution profiles from an `internal/profile` loads and validates execution profiles from an
operating-system filesystem or an `fs.FS`. It supports a primary repository operating-system filesystem or an `fs.FS`. It supports a primary repository
with fallback only when the primary reports that a profile is absent. with fallback only when the primary reports that a profile is absent. Strict
YAML decoding recognizes the optional `backend` field, trims its value, and
requires a model plus at least one non-blank backend or endpoint. Loading does
not check registry membership because the available registry belongs to the
assembled engine; the runner checks membership during preparation.
`internal/profile/builtin` embeds the maintained built-in profile catalog and `internal/profile/builtin` embeds the maintained built-in profile catalog and
can place a caller-selected repository ahead of that catalog. Profile behavior can place a caller-selected repository ahead of that catalog. Every embedded
is owned by the profile selects `openrouter` and inherits its endpoint and credential
environment-variable name from the built-in backend registry rather than
repeating those values. Profile behavior is owned by the
[profile repository tests](../../internal/profile/repository_test.go), while [profile repository tests](../../internal/profile/repository_test.go), while
catalog completeness, duplicate IDs, and overlay behavior are owned by the catalog completeness, backend-selection invariant, duplicate IDs, and overlay behavior are owned by the
[built-in repository tests](../../internal/profile/builtin/repository_test.go). [built-in repository tests](../../internal/profile/builtin/repository_test.go).
## Ordinary Artifacts ## Ordinary Artifacts

View File

@@ -299,6 +299,8 @@ immutable and the final repository race suite exercises concurrent reads.
## Stage 2 — Profile Selection And Effective Resolution ## Stage 2 — Profile Selection And Effective Resolution
**Status:** Complete.
### Goal ### Goal
Make the built-in OpenRouter backend selectable by file and in-memory profiles, Make the built-in OpenRouter backend selectable by file and in-memory profiles,

View File

@@ -12,6 +12,7 @@ import (
"time" "time"
artifactadapter "gitea.maximumdirect.net/eric/promptkit/internal/artifact" artifactadapter "gitea.maximumdirect.net/eric/promptkit/internal/artifact"
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/defaults" "gitea.maximumdirect.net/eric/promptkit/internal/defaults"
"gitea.maximumdirect.net/eric/promptkit/internal/llm" "gitea.maximumdirect.net/eric/promptkit/internal/llm"
"gitea.maximumdirect.net/eric/promptkit/internal/profile" "gitea.maximumdirect.net/eric/promptkit/internal/profile"
@@ -335,6 +336,11 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
profiles = profile.NewOverlayRepository(options.memoryProfiles, profiles) profiles = profile.NewOverlayRepository(options.memoryProfiles, profiles)
} }
backendRegistry, err := backend.NewRegistry(nil)
if err != nil {
return nil, fmt.Errorf("%w: failed to construct backend registry: %v", ErrInvalidConfig, err)
}
validator := options.validator validator := options.validator
if !options.validatorSource { if !options.validatorSource {
schemaDir := cfg.SchemaDir schemaDir := cfg.SchemaDir
@@ -365,6 +371,7 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
runner: usecase.NewRunner( runner: usecase.NewRunner(
promptDefs, promptDefs,
profiles, profiles,
backendRegistry,
artifacts, artifacts,
prompt.NewGoRenderer(), prompt.NewGoRenderer(),
llmClient, llmClient,

View File

@@ -1271,6 +1271,18 @@ func TestPrepareUsesBuiltInProfileWithoutProfileDir(t *testing.T) {
if prepared.SelectedProfileID != "mistral-small-3" { if prepared.SelectedProfileID != "mistral-small-3" {
t.Fatalf("unexpected selected profile: %q", prepared.SelectedProfileID) t.Fatalf("unexpected selected profile: %q", prepared.SelectedProfileID)
} }
if prepared.SelectedBackendID != promptkit.BackendOpenRouter {
t.Fatalf("unexpected selected backend: %q", prepared.SelectedBackendID)
}
if prepared.EffectiveModelParams.BackendID != promptkit.BackendOpenRouter {
t.Fatalf("unexpected effective backend: %q", prepared.EffectiveModelParams.BackendID)
}
if prepared.EffectiveModelParams.Endpoint != "https://openrouter.ai/api/v1" {
t.Fatalf("unexpected built-in endpoint: %q", prepared.EffectiveModelParams.Endpoint)
}
if prepared.EffectiveModelParams.APIKeyEnv != "OPENROUTER_API_KEY" {
t.Fatalf("unexpected built-in api key environment name: %q", prepared.EffectiveModelParams.APIKeyEnv)
}
if prepared.EffectiveModelParams.Model != "mistralai/mistral-small-3.2-24b-instruct" { if prepared.EffectiveModelParams.Model != "mistralai/mistral-small-3.2-24b-instruct" {
t.Fatalf("unexpected built-in model: %q", prepared.EffectiveModelParams.Model) t.Fatalf("unexpected built-in model: %q", prepared.EffectiveModelParams.Model)
} }
@@ -1641,6 +1653,7 @@ func TestOpenAICompatibleProfileRunsThroughNormalProfilePath(t *testing.T) {
fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}} fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
prof := promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{ prof := promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
ID: "template-profile", ID: "template-profile",
BackendID: " openrouter ",
Endpoint: "http://template/v1", Endpoint: "http://template/v1",
Model: "template-model", Model: "template-model",
APIKeyRequired: true, APIKeyRequired: true,
@@ -1672,7 +1685,9 @@ func TestOpenAICompatibleProfileRunsThroughNormalProfilePath(t *testing.T) {
if len(fake.requests) != 1 { if len(fake.requests) != 1 {
t.Fatalf("expected one request, got %d", len(fake.requests)) t.Fatalf("expected one request, got %d", len(fake.requests))
} }
if fake.requests[0].Target.Model != "template-model" || fake.requests[0].APIKey != "template-key" { if fake.requests[0].Target.BackendID != promptkit.BackendOpenRouter ||
fake.requests[0].Target.Model != "template-model" ||
fake.requests[0].APIKey != "template-key" {
t.Fatalf("unexpected generated request: %+v", fake.requests[0]) t.Fatalf("unexpected generated request: %+v", fake.requests[0])
} }
if !reflect.DeepEqual(fake.requests[0].Target.ExtraParams, map[string]any{"provider": "template"}) { if !reflect.DeepEqual(fake.requests[0].Target.ExtraParams, map[string]any{"provider": "template"}) {

View File

@@ -81,6 +81,7 @@ type RunResult struct {
PromptHash string PromptHash string
RenderedPromptHash string RenderedPromptHash string
SelectedProfileID string SelectedProfileID string
SelectedBackendID string
ModelName string ModelName string
Endpoint string Endpoint string
EffectiveModelParams ExecutionTarget EffectiveModelParams ExecutionTarget
@@ -98,6 +99,7 @@ type PreparedRun struct {
PromptVersion string `json:"prompt_version,omitempty"` PromptVersion string `json:"prompt_version,omitempty"`
PromptHash string `json:"prompt_hash,omitempty"` PromptHash string `json:"prompt_hash,omitempty"`
SelectedProfileID string `json:"selected_profile_id"` SelectedProfileID string `json:"selected_profile_id"`
SelectedBackendID string `json:"selected_backend_id,omitempty"`
EffectiveModelParams ExecutionTarget `json:"effective_model_params"` EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
TargetPresence ExecutionTargetPresence `json:"-"` TargetPresence ExecutionTargetPresence `json:"-"`
OutputContract OutputContract `json:"output_contract"` OutputContract OutputContract `json:"output_contract"`
@@ -168,6 +170,7 @@ type Backend struct {
// ExecutionProfile describes how and where to execute a model. // ExecutionProfile describes how and where to execute a model.
type ExecutionProfile struct { type ExecutionProfile struct {
ID string `yaml:"id"` ID string `yaml:"id"`
BackendID string `yaml:"backend"`
Endpoint string `yaml:"endpoint"` Endpoint string `yaml:"endpoint"`
Model string `yaml:"model"` Model string `yaml:"model"`
Temperature float64 `yaml:"temperature"` Temperature float64 `yaml:"temperature"`
@@ -206,6 +209,7 @@ type ExecutionTargetPresence struct {
// ExecutionTarget represents effective model runtime settings for a run. // ExecutionTarget represents effective model runtime settings for a run.
type ExecutionTarget struct { type ExecutionTarget struct {
BackendID string `yaml:"backend" json:"backend_id,omitempty"`
Endpoint string `yaml:"endpoint" json:"endpoint"` Endpoint string `yaml:"endpoint" json:"endpoint"`
Model string `yaml:"model" json:"model"` Model string `yaml:"model" json:"model"`
Temperature float64 `yaml:"temperature" json:"temperature"` Temperature float64 `yaml:"temperature" json:"temperature"`

View File

@@ -1,9 +1,8 @@
id: aion-2 id: aion-2
endpoint: https://openrouter.ai/api/v1 backend: openrouter
model: aion-labs/aion-2.0 model: aion-labs/aion-2.0
temperature: 0.72 temperature: 0.72
reasoning_effort: high reasoning_effort: high
top_p: 0.95 top_p: 0.95
timeout_seconds: 180 timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex service_tier: flex

View File

@@ -1,7 +1,6 @@
id: claude-fable-latest id: claude-fable-latest
endpoint: https://openrouter.ai/api/v1 backend: openrouter
model: "~anthropic/claude-fable-latest" model: "~anthropic/claude-fable-latest"
reasoning_effort: high reasoning_effort: high
timeout_seconds: 600 timeout_seconds: 600
api_key_env: OPENROUTER_API_KEY
service_tier: flex service_tier: flex

View File

@@ -1,7 +1,6 @@
id: claude-haiku-latest id: claude-haiku-latest
endpoint: https://openrouter.ai/api/v1 backend: openrouter
model: "~anthropic/claude-haiku-latest" model: "~anthropic/claude-haiku-latest"
reasoning_effort: medium reasoning_effort: medium
timeout_seconds: 240 timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex service_tier: flex

View File

@@ -1,7 +1,6 @@
id: claude-opus-latest id: claude-opus-latest
endpoint: https://openrouter.ai/api/v1 backend: openrouter
model: "~anthropic/claude-opus-latest" model: "~anthropic/claude-opus-latest"
reasoning_effort: high reasoning_effort: high
timeout_seconds: 240 timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex service_tier: flex

View File

@@ -1,7 +1,6 @@
id: claude-sonnet-latest id: claude-sonnet-latest
endpoint: https://openrouter.ai/api/v1 backend: openrouter
model: "~anthropic/claude-sonnet-latest" model: "~anthropic/claude-sonnet-latest"
reasoning_effort: high reasoning_effort: high
timeout_seconds: 240 timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex service_tier: flex

View File

@@ -1,7 +1,6 @@
id: deepseek-3-2 id: deepseek-3-2
endpoint: https://openrouter.ai/api/v1 backend: openrouter
model: deepseek/deepseek-v3.2 model: deepseek/deepseek-v3.2
reasoning_effort: high reasoning_effort: high
timeout_seconds: 180 timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex service_tier: flex

View File

@@ -1,7 +1,6 @@
id: deepseek-4-flash id: deepseek-4-flash
endpoint: https://openrouter.ai/api/v1 backend: openrouter
model: deepseek/deepseek-v4-flash model: deepseek/deepseek-v4-flash
#reasoning_effort: medium #reasoning_effort: medium
timeout_seconds: 180 timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex service_tier: flex

View File

@@ -1,7 +1,6 @@
id: deepseek-4-pro id: deepseek-4-pro
endpoint: https://openrouter.ai/api/v1 backend: openrouter
model: deepseek/deepseek-v4-pro model: deepseek/deepseek-v4-pro
reasoning_effort: high reasoning_effort: high
timeout_seconds: 180 timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex service_tier: flex

View File

@@ -1,9 +1,8 @@
id: gemini-2-flash-lite id: gemini-2-flash-lite
endpoint: https://openrouter.ai/api/v1 backend: openrouter
model: "google/gemini-2.5-flash-lite" model: "google/gemini-2.5-flash-lite"
#temperature: 0.15 #temperature: 0.15
reasoning_effort: high reasoning_effort: high
#top_p: 0.98 #top_p: 0.98
timeout_seconds: 240 timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex service_tier: flex

View File

@@ -1,9 +1,8 @@
id: gemini-2-flash id: gemini-2-flash
endpoint: https://openrouter.ai/api/v1 backend: openrouter
model: "google/gemini-2.5-flash" model: "google/gemini-2.5-flash"
#temperature: 0.15 #temperature: 0.15
reasoning_effort: high reasoning_effort: high
#top_p: 0.98 #top_p: 0.98
timeout_seconds: 240 timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex service_tier: flex

View File

@@ -1,9 +1,8 @@
id: gemini-2-pro id: gemini-2-pro
endpoint: https://openrouter.ai/api/v1 backend: openrouter
model: "google/gemini-2.5-pro" model: "google/gemini-2.5-pro"
#temperature: 0.15 #temperature: 0.15
reasoning_effort: high reasoning_effort: high
#top_p: 0.98 #top_p: 0.98
timeout_seconds: 240 timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex service_tier: flex

View File

@@ -1,9 +1,8 @@
id: gemini-3-flash-lite id: gemini-3-flash-lite
endpoint: https://openrouter.ai/api/v1 backend: openrouter
model: "google/gemini-3.1-flash-lite" model: "google/gemini-3.1-flash-lite"
#temperature: 0.15 #temperature: 0.15
reasoning_effort: high reasoning_effort: high
#top_p: 0.98 #top_p: 0.98
timeout_seconds: 240 timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex service_tier: flex

View File

@@ -1,9 +1,8 @@
id: gemini-flash-latest id: gemini-flash-latest
endpoint: https://openrouter.ai/api/v1 backend: openrouter
model: "~google/gemini-flash-latest" model: "~google/gemini-flash-latest"
#temperature: 0.15 #temperature: 0.15
reasoning_effort: high reasoning_effort: high
#top_p: 0.98 #top_p: 0.98
timeout_seconds: 240 timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex service_tier: flex

View File

@@ -1,9 +1,8 @@
id: gemini-pro-latest id: gemini-pro-latest
endpoint: https://openrouter.ai/api/v1 backend: openrouter
model: "~google/gemini-pro-latest" model: "~google/gemini-pro-latest"
#temperature: 0.15 #temperature: 0.15
reasoning_effort: high reasoning_effort: high
#top_p: 0.98 #top_p: 0.98
timeout_seconds: 240 timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex service_tier: flex

View File

@@ -1,9 +1,8 @@
id: gemma-4-31b id: gemma-4-31b
endpoint: https://openrouter.ai/api/v1 backend: openrouter
model: google/gemma-4-31b-it:exacto model: google/gemma-4-31b-it:exacto
temperature: 0.15 temperature: 0.15
reasoning_effort: high reasoning_effort: high
top_p: 0.98 top_p: 0.98
timeout_seconds: 240 timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex service_tier: flex

View File

@@ -1,9 +1,8 @@
id: minimax-m2 id: minimax-m2
endpoint: https://openrouter.ai/api/v1 backend: openrouter
model: minimax/minimax-m2.5 model: minimax/minimax-m2.5
temperature: 0.5 temperature: 0.5
reasoning_effort: high reasoning_effort: high
top_p: 0.95 top_p: 0.95
timeout_seconds: 180 timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex service_tier: flex

View File

@@ -1,9 +1,8 @@
id: minimax-m3 id: minimax-m3
endpoint: https://openrouter.ai/api/v1 backend: openrouter
model: minimax/minimax-m3 model: minimax/minimax-m3
#temperature: 0.5 #temperature: 0.5
reasoning_effort: high reasoning_effort: high
#top_p: 0.95 #top_p: 0.95
timeout_seconds: 180 timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex service_tier: flex

View File

@@ -1,7 +1,6 @@
id: mistral-large-2512 id: mistral-large-2512
endpoint: https://openrouter.ai/api/v1 backend: openrouter
model: mistralai/mistral-large-2512 model: mistralai/mistral-large-2512
temperature: 0.15 temperature: 0.15
top_p: 0.98 top_p: 0.98
timeout_seconds: 180 timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY

View File

@@ -1,8 +1,7 @@
id: mistral-medium-3-5 id: mistral-medium-3-5
endpoint: https://openrouter.ai/api/v1 backend: openrouter
model: mistralai/mistral-medium-3-5 model: mistralai/mistral-medium-3-5
temperature: 0.15 temperature: 0.15
reasoning_effort: high reasoning_effort: high
top_p: 0.98 top_p: 0.98
timeout_seconds: 180 timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY

View File

@@ -1,7 +1,6 @@
id: mistral-small-3 id: mistral-small-3
endpoint: https://openrouter.ai/api/v1 backend: openrouter
model: mistralai/mistral-small-3.2-24b-instruct model: mistralai/mistral-small-3.2-24b-instruct
temperature: 0.05 temperature: 0.05
top_p: 1.0 top_p: 1.0
timeout_seconds: 180 timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY

View File

@@ -1,8 +1,7 @@
id: mistral-small-4 id: mistral-small-4
endpoint: https://openrouter.ai/api/v1 backend: openrouter
model: mistralai/mistral-small-2603 model: mistralai/mistral-small-2603
temperature: 0.1 temperature: 0.1
reasoning_effort: high reasoning_effort: high
top_p: 0.98 top_p: 0.98
timeout_seconds: 180 timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY

View File

@@ -1,7 +1,6 @@
id: nemotron-3-ultra id: nemotron-3-ultra
endpoint: https://openrouter.ai/api/v1 backend: openrouter
model: nvidia/nemotron-3-ultra-550b-a55b model: nvidia/nemotron-3-ultra-550b-a55b
reasoning_effort: high reasoning_effort: high
timeout_seconds: 180 timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex service_tier: flex

View File

@@ -1,7 +1,6 @@
id: gpt-5-mini id: gpt-5-mini
endpoint: https://openrouter.ai/api/v1 backend: openrouter
model: "openai/gpt-5.4-mini" model: "openai/gpt-5.4-mini"
reasoning_effort: high reasoning_effort: high
timeout_seconds: 240 timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex service_tier: flex

View File

@@ -1,7 +1,6 @@
id: gpt-5-nano id: gpt-5-nano
endpoint: https://openrouter.ai/api/v1 backend: openrouter
model: "openai/gpt-5.4-nano" model: "openai/gpt-5.4-nano"
reasoning_effort: high reasoning_effort: high
timeout_seconds: 240 timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex service_tier: flex

View File

@@ -7,6 +7,7 @@ import (
"strings" "strings"
"testing" "testing"
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/domain" "gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/profile" "gitea.maximumdirect.net/eric/promptkit/internal/profile"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
@@ -28,6 +29,12 @@ func TestBuiltInProfilesValidateThroughRepository(t *testing.T) {
if p.ID != id { if p.ID != id {
t.Fatalf("expected profile id %q, got %q", id, p.ID) t.Fatalf("expected profile id %q, got %q", id, p.ID)
} }
if p.BackendID != backend.OpenRouterID {
t.Fatalf("expected profile %q to select %q, got %q", id, backend.OpenRouterID, p.BackendID)
}
if p.Endpoint != "" || p.APIKeyEnv != "" {
t.Fatalf("expected profile %q to inherit backend connection settings, got endpoint=%q api_key_env=%q", id, p.Endpoint, p.APIKeyEnv)
}
}) })
} }
} }
@@ -60,6 +67,15 @@ func loadBuiltInProfileIDs(t *testing.T) map[string]string {
if _, ok := raw["api_key"]; ok { if _, ok := raw["api_key"]; ok {
t.Fatalf("built-in profile %s contains raw api_key", name) t.Fatalf("built-in profile %s contains raw api_key", name)
} }
if raw["backend"] != backend.OpenRouterID {
t.Fatalf("built-in profile %s does not select %q", name, backend.OpenRouterID)
}
if _, ok := raw["endpoint"]; ok {
t.Fatalf("built-in profile %s repeats endpoint", name)
}
if _, ok := raw["api_key_env"]; ok {
t.Fatalf("built-in profile %s repeats api_key_env", name)
}
id, ok := raw["id"].(string) id, ok := raw["id"].(string)
if !ok || strings.TrimSpace(id) == "" { if !ok || strings.TrimSpace(id) == "" {
t.Fatalf("built-in profile %s has missing id", name) t.Fatalf("built-in profile %s has missing id", name)

View File

@@ -121,6 +121,7 @@ func loadProfile(ctx context.Context, fsys fs.FS, root string, id string) (*doma
if prof.ID != id { if prof.ID != id {
continue continue
} }
prof.BackendID = strings.TrimSpace(prof.BackendID)
if err := validateProfile(&prof); err != nil { if err := validateProfile(&prof); err != nil {
if errors.Is(err, ErrRawAPIKeyNotAllowed) { if errors.Is(err, ErrRawAPIKeyNotAllowed) {
return nil, fmt.Errorf("%w: %s", err, relPath) return nil, fmt.Errorf("%w: %s", err, relPath)
@@ -189,8 +190,8 @@ func validateProfile(p *domain.ExecutionProfile) error {
if strings.TrimSpace(p.ID) == "" { if strings.TrimSpace(p.ID) == "" {
return errors.New("id is required") return errors.New("id is required")
} }
if strings.TrimSpace(p.Endpoint) == "" { if strings.TrimSpace(p.BackendID) == "" && strings.TrimSpace(p.Endpoint) == "" {
return errors.New("endpoint is required") return errors.New("backend or endpoint is required")
} }
if strings.TrimSpace(p.Model) == "" { if strings.TrimSpace(p.Model) == "" {
return errors.New("model is required") return errors.New("model is required")

View File

@@ -52,6 +52,43 @@ func TestFilesystemRepository_GetProfile(t *testing.T) {
} }
}) })
t.Run("backend and endpoint connection matrix", func(t *testing.T) {
tests := []struct {
name string
connection string
wantBackend string
wantEndpoint string
wantErr bool
}{
{name: "backend only", connection: "backend: ' openrouter '", wantBackend: "openrouter"},
{name: "endpoint only", connection: "endpoint: http://localhost:8000/v1", wantEndpoint: "http://localhost:8000/v1"},
{name: "both", connection: "backend: openrouter\nendpoint: http://localhost:8000/v1", wantBackend: "openrouter", wantEndpoint: "http://localhost:8000/v1"},
{name: "neither", wantErr: true},
{name: "blank backend", connection: "backend: ' '", wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
id := "connection-" + strings.ReplaceAll(tt.name, " ", "-")
writeProfileTestFile(t, filepath.Join(tmpDir, id+".yaml"), "id: "+id+"\nmodel: model\n"+tt.connection+"\n")
p, err := repo.GetProfile(ctx, id)
if tt.wantErr {
if !errors.Is(err, ErrInvalidProfile) {
t.Fatalf("expected ErrInvalidProfile, got %v", err)
}
return
}
if err != nil {
t.Fatalf("expected profile to load, got %v", err)
}
if p.BackendID != tt.wantBackend || p.Endpoint != tt.wantEndpoint {
t.Fatalf("unexpected connection values: backend=%q endpoint=%q", p.BackendID, p.Endpoint)
}
})
}
})
t.Run("valid profile with api_key_env", func(t *testing.T) { t.Run("valid profile with api_key_env", func(t *testing.T) {
p, err := repo.GetProfile(ctx, "local-secure") p, err := repo.GetProfile(ctx, "local-secure")
if err != nil { if err != nil {

View File

@@ -40,6 +40,7 @@ var (
type Runner struct { type Runner struct {
promptDefs promptdef.Repository promptDefs promptdef.Repository
profiles profile.Repository profiles profile.Repository
backends BackendResolver
artifacts artifact.Reader artifacts artifact.Reader
renderer prompt.Renderer renderer prompt.Renderer
llm llm.Client llm llm.Client
@@ -47,20 +48,27 @@ type Runner struct {
repairer OutputRepairer repairer OutputRepairer
} }
// BackendResolver resolves one normalized backend ID.
type BackendResolver interface {
GetBackend(string) (domain.Backend, error)
}
func NewRunner( func NewRunner(
promptDefs promptdef.Repository, promptDefs promptdef.Repository,
profiles profile.Repository, profiles profile.Repository,
backends BackendResolver,
artifacts artifact.Reader, artifacts artifact.Reader,
renderer prompt.Renderer, renderer prompt.Renderer,
llmClient llm.Client, llmClient llm.Client,
validator validate.Validator, validator validate.Validator,
) *Runner { ) *Runner {
return NewRunnerWithRepairer(promptDefs, profiles, artifacts, renderer, llmClient, validator, nil) return NewRunnerWithRepairer(promptDefs, profiles, backends, artifacts, renderer, llmClient, validator, nil)
} }
func NewRunnerWithRepairer( func NewRunnerWithRepairer(
promptDefs promptdef.Repository, promptDefs promptdef.Repository,
profiles profile.Repository, profiles profile.Repository,
backends BackendResolver,
artifacts artifact.Reader, artifacts artifact.Reader,
renderer prompt.Renderer, renderer prompt.Renderer,
llmClient llm.Client, llmClient llm.Client,
@@ -70,6 +78,7 @@ func NewRunnerWithRepairer(
return &Runner{ return &Runner{
promptDefs: promptDefs, promptDefs: promptDefs,
profiles: profiles, profiles: profiles,
backends: backends,
artifacts: artifacts, artifacts: artifacts,
renderer: renderer, renderer: renderer,
llm: llmClient, llm: llmClient,
@@ -153,6 +162,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
PromptHash: prepared.PromptHash, PromptHash: prepared.PromptHash,
RenderedPromptHash: prepared.RenderedPromptHash, RenderedPromptHash: prepared.RenderedPromptHash,
SelectedProfileID: prepared.SelectedProfileID, SelectedProfileID: prepared.SelectedProfileID,
SelectedBackendID: prepared.SelectedBackendID,
ModelName: prepared.EffectiveModelParams.Model, ModelName: prepared.EffectiveModelParams.Model,
Endpoint: prepared.EffectiveModelParams.Endpoint, Endpoint: prepared.EffectiveModelParams.Endpoint,
EffectiveModelParams: prepared.EffectiveModelParams, EffectiveModelParams: prepared.EffectiveModelParams,
@@ -193,7 +203,20 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err) return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
} }
effectiveModel, targetPresence, err := resolveExecutionTarget(execProfile, req.Execution) var selectedBackend *domain.Backend
if backendID := strings.TrimSpace(execProfile.BackendID); backendID != "" {
execProfile.BackendID = backendID
if r.backends == nil {
return nil, fmt.Errorf("%w: backend %q cannot be resolved", ErrProfileLoad, backendID)
}
resolvedBackend, resolveErr := r.backends.GetBackend(backendID)
if resolveErr != nil {
return nil, fmt.Errorf("%w: backend %q: %w", ErrProfileLoad, backendID, resolveErr)
}
selectedBackend = &resolvedBackend
}
effectiveModel, targetPresence, err := resolveExecutionTarget(selectedBackend, execProfile, req.Execution)
if err != nil { if err != nil {
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err) return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
} }
@@ -239,6 +262,7 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
PromptVersion: def.Version, PromptVersion: def.Version,
PromptHash: promptDefinitionHash, PromptHash: promptDefinitionHash,
SelectedProfileID: selectedProfileID, SelectedProfileID: selectedProfileID,
SelectedBackendID: effectiveModel.BackendID,
EffectiveModelParams: effectiveModel, EffectiveModelParams: effectiveModel,
TargetPresence: targetPresence, TargetPresence: targetPresence,
OutputContract: effectiveContract, OutputContract: effectiveContract,
@@ -338,6 +362,9 @@ func (r *Runner) shouldAttemptRepair(contract domain.OutputContract, validationR
func mergeExecutionTarget(base domain.ExecutionTarget, override domain.ExecutionTarget) domain.ExecutionTarget { func mergeExecutionTarget(base domain.ExecutionTarget, override domain.ExecutionTarget) domain.ExecutionTarget {
out := base out := base
if strings.TrimSpace(override.BackendID) != "" {
out.BackendID = override.BackendID
}
if override.Endpoint != "" { if override.Endpoint != "" {
out.Endpoint = override.Endpoint out.Endpoint = override.Endpoint
} }
@@ -367,6 +394,7 @@ func mergeExecutionTarget(base domain.ExecutionTarget, override domain.Execution
} }
if override.APIKeyRequired { if override.APIKeyRequired {
out.APIKeyRequired = true out.APIKeyRequired = true
out.APIKeyEnv = ""
} }
if len(override.ExtraParams) > 0 { if len(override.ExtraParams) > 0 {
out.ExtraParams = copyExtraParams(override.ExtraParams) out.ExtraParams = copyExtraParams(override.ExtraParams)
@@ -426,8 +454,9 @@ func mergeExecutionTargetOverride(base domain.ExecutionTarget, override domain.E
return out, presence, nil return out, presence, nil
} }
func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) { func resolveExecutionTarget(backendValue *domain.Backend, profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) {
out := defaults.ExecutionTargetDefault() out := defaults.ExecutionTargetDefault()
out = mergeExecutionTarget(out, backendToTarget(backendValue))
out = mergeExecutionTarget(out, executionProfileToTarget(profileValue)) out = mergeExecutionTarget(out, executionProfileToTarget(profileValue))
var presence domain.ExecutionTargetPresence var presence domain.ExecutionTargetPresence
if override != nil { if override != nil {
@@ -461,8 +490,13 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
if p == nil { if p == nil {
return domain.ExecutionTarget{} return domain.ExecutionTarget{}
} }
endpoint := p.Endpoint
if strings.TrimSpace(endpoint) == "" {
endpoint = ""
}
return domain.ExecutionTarget{ return domain.ExecutionTarget{
Endpoint: p.Endpoint, BackendID: p.BackendID,
Endpoint: endpoint,
Model: p.Model, Model: p.Model,
Temperature: p.Temperature, Temperature: p.Temperature,
MaxTokens: p.MaxTokens, MaxTokens: p.MaxTokens,
@@ -476,6 +510,18 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
} }
} }
func backendToTarget(value *domain.Backend) domain.ExecutionTarget {
if value == nil {
return domain.ExecutionTarget{}
}
return domain.ExecutionTarget{
BackendID: value.ID,
Endpoint: value.Endpoint,
APIKeyEnv: value.APIKeyEnv,
ExtraParams: copyExtraParams(value.ExtraParams),
}
}
func copyExtraParams(src map[string]any) map[string]any { func copyExtraParams(src map[string]any) map[string]any {
if len(src) == 0 { if len(src) == 0 {
return nil return nil

View File

@@ -34,6 +34,18 @@ type fakeExecutionProfileRepo struct {
lastID string lastID string
} }
type fakeBackendResolver struct {
backends map[string]domain.Backend
}
func (f fakeBackendResolver) GetBackend(id string) (domain.Backend, error) {
value, ok := f.backends[id]
if !ok {
return domain.Backend{}, errors.New("backend not found")
}
return value, nil
}
func (f *fakeExecutionProfileRepo) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) { func (f *fakeExecutionProfileRepo) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
f.lastID = id f.lastID = id
if f.err != nil { if f.err != nil {
@@ -167,7 +179,7 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) {
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}} renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
llmClient := &fakeLLM{forbid: true} llmClient := &fakeLLM{forbid: true}
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil) runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p", PromptID: "p",
PromptVersion: "1", PromptVersion: "1",
@@ -257,7 +269,7 @@ func TestRunnerPrepareSelectedProfileDoesNotExistFails(t *testing.T) {
} }
func TestRunnerPreparePromptLoadFailure(t *testing.T) { func TestRunnerPreparePromptLoadFailure(t *testing.T) {
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil) runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p"}) _, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p"})
if !errors.Is(err, ErrPromptLoad) { if !errors.Is(err, ErrPromptLoad) {
t.Fatalf("expected ErrPromptLoad, got %v", err) t.Fatalf("expected ErrPromptLoad, got %v", err)
@@ -281,7 +293,7 @@ func TestRunnerPrepareRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) {
ServiceTier: "priority", ServiceTier: "priority",
}, },
}} }}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil) runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p", PromptID: "p",
@@ -379,12 +391,12 @@ func TestRunnerPrepareRequestNumericOverridePresence(t *testing.T) {
TopP: 0.8, TopP: 0.8,
TimeoutSeconds: 45, TimeoutSeconds: 45,
}, },
}}, }}, nil,
defaultArtifactReader(), defaultArtifactReader(),
defaultRenderer(), defaultRenderer(),
&fakeLLM{forbid: true}, &fakeLLM{forbid: true},
nil, nil)
)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p", PromptID: "p",
@@ -426,12 +438,12 @@ func TestRunnerPrepareInvalidRequestNumericOverridesFail(t *testing.T) {
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
runner := NewRunner( runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(), defaultArtifactReader(),
defaultRenderer(), defaultRenderer(),
&fakeLLM{forbid: true}, &fakeLLM{forbid: true},
nil, nil)
)
_, err := runner.Prepare(context.Background(), domain.RunRequest{ _, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p", PromptID: "p",
@@ -458,7 +470,7 @@ func TestRunnerPrepareSelectedProfileBeatsBuiltInDefault(t *testing.T) {
ServiceTier: "priority", ServiceTier: "priority",
}, },
}} }}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil) runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p", PromptID: "p",
@@ -495,12 +507,12 @@ func TestRunnerPrepareFileBackedPromptBodiesRenderCorrectly(t *testing.T) {
llmClient := &fakeLLM{forbid: true} llmClient := &fakeLLM{forbid: true}
runner := NewRunner( runner := NewRunner(
promptdef.NewFilesystemRepository(promptDir), promptdef.NewFilesystemRepository(promptDir),
profile.NewFilesystemRepository(profileDir), profile.NewFilesystemRepository(profileDir), nil,
reader, reader,
prompt.NewGoRenderer(), prompt.NewGoRenderer(),
llmClient, llmClient,
nil, nil)
)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "valid-file-backed", PromptID: "valid-file-backed",
@@ -526,12 +538,13 @@ func TestRunnerPrepareRequiredInputMissingFails(t *testing.T) {
runner := NewRunner( runner := NewRunner(
&fakePromptRepo{def: def}, &fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(), defaultArtifactReader(),
prompt.NewGoRenderer(), prompt.NewGoRenderer(),
&fakeLLM{forbid: true}, &fakeLLM{forbid: true},
nil, nil)
)
_, err := runner.Prepare(context.Background(), domain.RunRequest{ _, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p", PromptID: "p",
ProfileID: "exec", ProfileID: "exec",
@@ -552,12 +565,13 @@ func TestRunnerPrepareUnknownTemplateInputReferenceFails(t *testing.T) {
runner := NewRunner( runner := NewRunner(
&fakePromptRepo{def: def}, &fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(), defaultArtifactReader(),
prompt.NewGoRenderer(), prompt.NewGoRenderer(),
&fakeLLM{forbid: true}, &fakeLLM{forbid: true},
nil, nil)
)
_, err := runner.Prepare(context.Background(), domain.RunRequest{ _, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p", PromptID: "p",
ProfileID: "exec", ProfileID: "exec",
@@ -579,7 +593,7 @@ func TestRunnerPrepareAPIKeyEnvNameIncludedButNotResolvedValue(t *testing.T) {
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName}, "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName},
}} }}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil) runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()}) prepared, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if err != nil { if err != nil {
@@ -607,12 +621,12 @@ func TestRunnerPrepareJSONSchemaBuildsStructuredOutputSpec(t *testing.T) {
} }
runner := NewRunner( runner := NewRunner(
&fakePromptRepo{def: def}, &fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(), defaultArtifactReader(),
defaultRenderer(), defaultRenderer(),
&fakeLLM{forbid: true}, &fakeLLM{forbid: true},
validator, validator)
)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{ prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p", PromptID: "p",
@@ -651,12 +665,12 @@ func TestRunnerPrepareJSONSchemaSchemaLoadFailureReturnsValidationError(t *testi
validator := &fakeValidator{schemaErr: errors.New("schema unavailable")} validator := &fakeValidator{schemaErr: errors.New("schema unavailable")}
runner := NewRunner( runner := NewRunner(
&fakePromptRepo{def: def}, &fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(), defaultArtifactReader(),
defaultRenderer(), defaultRenderer(),
&fakeLLM{forbid: true}, &fakeLLM{forbid: true},
validator, validator)
)
_, err := runner.Prepare(context.Background(), domain.RunRequest{ _, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p", PromptID: "p",
@@ -681,12 +695,12 @@ func TestRunnerRunJSONSchemaSchemaLoadFailureFailsBeforeLLM(t *testing.T) {
validator := &fakeValidator{schemaErr: errors.New("schema unavailable")} validator := &fakeValidator{schemaErr: errors.New("schema unavailable")}
runner := NewRunner( runner := NewRunner(
&fakePromptRepo{def: def}, &fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(), defaultArtifactReader(),
defaultRenderer(), defaultRenderer(),
llmClient, llmClient,
validator, validator)
)
_, err := runner.Run(context.Background(), domain.RunRequest{ _, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p", PromptID: "p",
@@ -836,7 +850,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}} renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}}
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil) runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{ res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p", PromptID: "p",
PromptVersion: "1", PromptVersion: "1",
@@ -915,7 +929,7 @@ func TestRunnerRunPassesExtraParamsToGenerateRequestTarget(t *testing.T) {
}, },
}} }}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil) runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{ res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p", PromptID: "p",
@@ -941,7 +955,7 @@ func TestRunnerRunAndPrepareResolveSameProfileAndEffectiveSettings(t *testing.T)
}} }}
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}} renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap"}} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap"}}
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil) runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil)
req := domain.RunRequest{ req := domain.RunRequest{
PromptID: "p", PromptID: "p",
@@ -1071,7 +1085,7 @@ func TestRunnerRunExplicitRuntimeOverrideBeatsSelectedProfileValue(t *testing.T)
}, },
}} }}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil) runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{ res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p", PromptID: "p",
@@ -1115,7 +1129,7 @@ func TestRunnerRunSelectedProfileBeatsBuiltInDefault(t *testing.T) {
}, },
}} }}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil) runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{ res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p", PromptID: "p",
@@ -1142,7 +1156,7 @@ func TestRunnerRunBuiltInDefaultsUsedWhenProfileOmitsOptionalFields(t *testing.T
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"}, "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"},
}} }}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil) runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{ res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p", PromptID: "p",
@@ -1172,7 +1186,7 @@ func TestRunnerRunAPIKeyEnvResolvesFromEnvironment(t *testing.T) {
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_TEST_API_KEY"}, "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_TEST_API_KEY"},
}} }}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil) runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()}) res, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if err != nil { if err != nil {
@@ -1188,7 +1202,7 @@ func TestRunnerRunAPIKeyEnvMissingEnvironmentValueFailsClearly(t *testing.T) {
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_MISSING_KEY"}, "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_MISSING_KEY"},
}} }}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil) runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()}) _, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if !errors.Is(err, ErrInvalidRequest) { if !errors.Is(err, ErrInvalidRequest) {
@@ -1209,7 +1223,7 @@ func TestRunnerRunDirectAPIKeyBypassesMissingEnvAndReachesLLM(t *testing.T) {
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_MISSING_KEY"}, "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_MISSING_KEY"},
}} }}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil) runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{ _, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p", PromptID: "p",
@@ -1233,7 +1247,7 @@ func TestRunnerPrepareAPIKeyRequiredFailsWithoutDirectKey(t *testing.T) {
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true}, "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true},
}} }}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil) runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{ _, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p", PromptID: "p",
@@ -1252,7 +1266,7 @@ func TestRunnerRunAPIKeyRequiredSucceedsWithDirectKey(t *testing.T) {
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true}, "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true},
}} }}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil) runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{ _, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p", PromptID: "p",
@@ -1279,7 +1293,7 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideWorks(t *testing.T) {
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"}, "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"},
}} }}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil) runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{ res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p", PromptID: "p",
@@ -1304,7 +1318,7 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideBeatsProfile(t *testing.T) {
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: profileEnv}, "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: profileEnv},
}} }}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil) runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{ res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p", PromptID: "p",
@@ -1328,7 +1342,7 @@ func TestRunnerRunAPIKeyValueNotPresentInMetadata(t *testing.T) {
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName}, "exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName},
}} }}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil) runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()}) res, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if err != nil { if err != nil {
@@ -1344,7 +1358,7 @@ func TestRunnerRunAPIKeyValueNotPresentInMetadata(t *testing.T) {
} }
func TestRunnerRunPromptLoadFailure(t *testing.T) { func TestRunnerRunPromptLoadFailure(t *testing.T) {
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil) runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"}) _, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"})
if !errors.Is(err, ErrPromptLoad) { if !errors.Is(err, ErrPromptLoad) {
t.Fatalf("expected ErrPromptLoad, got %v", err) t.Fatalf("expected ErrPromptLoad, got %v", err)
@@ -1357,12 +1371,12 @@ func TestRunnerRunPromptLoadFailure(t *testing.T) {
func TestRunnerRunArtifactLoadFailure(t *testing.T) { func TestRunnerRunArtifactLoadFailure(t *testing.T) {
runner := NewRunner( runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
&fakeArtifactReader{errByURI: map[string]error{"a://bad": errors.New("read failed")}}, &fakeArtifactReader{errByURI: map[string]error{"a://bad": errors.New("read failed")}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}}, &fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
nil, nil)
)
_, err := runner.Run(context.Background(), domain.RunRequest{ _, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p", PromptID: "p",
@@ -1377,12 +1391,13 @@ func TestRunnerRunArtifactLoadFailure(t *testing.T) {
func TestRunnerRunPromptRenderFailure(t *testing.T) { func TestRunnerRunPromptRenderFailure(t *testing.T) {
runner := NewRunner( runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(), defaultArtifactReader(),
&fakeRenderer{err: errors.New("render failed")}, &fakeRenderer{err: errors.New("render failed")},
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
nil, nil)
)
_, err := runner.Run(context.Background(), domain.RunRequest{ _, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p", PromptID: "p",
ProfileID: "exec", ProfileID: "exec",
@@ -1396,12 +1411,13 @@ func TestRunnerRunPromptRenderFailure(t *testing.T) {
func TestRunnerRunLLMFailure(t *testing.T) { func TestRunnerRunLLMFailure(t *testing.T) {
runner := NewRunner( runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(), defaultArtifactReader(),
defaultRenderer(), defaultRenderer(),
&fakeLLM{err: errors.New("llm failed")}, &fakeLLM{err: errors.New("llm failed")},
nil, nil)
)
_, err := runner.Run(context.Background(), domain.RunRequest{ _, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p", PromptID: "p",
ProfileID: "exec", ProfileID: "exec",
@@ -1415,12 +1431,13 @@ func TestRunnerRunLLMFailure(t *testing.T) {
func TestRunnerRunCancellationPreservesGenerationCategory(t *testing.T) { func TestRunnerRunCancellationPreservesGenerationCategory(t *testing.T) {
runner := NewRunner( runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(), defaultArtifactReader(),
defaultRenderer(), defaultRenderer(),
&fakeLLM{resp: &domain.GenerateResponse{Content: "ignored"}}, &fakeLLM{resp: &domain.GenerateResponse{Content: "ignored"}},
nil, nil)
)
ctx, cancel := context.WithCancel(context.Background()) ctx, cancel := context.WithCancel(context.Background())
cancel() cancel()
@@ -1440,12 +1457,13 @@ func TestRunnerRunCancellationPreservesGenerationCategory(t *testing.T) {
func TestRunnerRunLLMInvalidRequestMapsToUsecaseInvalidRequest(t *testing.T) { func TestRunnerRunLLMInvalidRequestMapsToUsecaseInvalidRequest(t *testing.T) {
runner := NewRunner( runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}, &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(), defaultArtifactReader(),
defaultRenderer(), defaultRenderer(),
&fakeLLM{err: llm.ErrInvalidRequest}, &fakeLLM{err: llm.ErrInvalidRequest},
nil, nil)
)
_, err := runner.Run(context.Background(), domain.RunRequest{ _, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p", PromptID: "p",
ProfileID: "exec", ProfileID: "exec",
@@ -1463,12 +1481,13 @@ func TestRunnerRunValidationStillWorks(t *testing.T) {
validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}} validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}}
runner := NewRunner( runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)}, &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(), defaultArtifactReader(),
defaultRenderer(), defaultRenderer(),
&fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}}, &fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}},
validator, validator)
)
res, err := runner.Run(context.Background(), domain.RunRequest{ res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p", PromptID: "p",
ProfileID: "exec", ProfileID: "exec",
@@ -1489,14 +1508,17 @@ func TestRunnerRunStructuredRepairRemainsBoundedAndUsesEffectiveModelSettings(t
runner := NewRunnerWithRepairer( runner := NewRunnerWithRepairer(
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)}, &fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{ &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", TimeoutSeconds: 55}, "exec": {ID: "exec", BackendID: "custom", Model: "profile-model", TimeoutSeconds: 55},
}}, fakeBackendResolver{backends: map[string]domain.Backend{
"custom": {ID: "custom", Endpoint: "http://backend/v1"},
}}, }},
defaultArtifactReader(), defaultArtifactReader(),
defaultRenderer(), defaultRenderer(),
llmClient, llmClient,
validate.NewStandardValidator("."), validate.NewStandardValidator("."),
repairer, repairer)
)
res, err := runner.Run(context.Background(), domain.RunRequest{ res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p", PromptID: "p",
ProfileID: "exec", ProfileID: "exec",
@@ -1518,6 +1540,12 @@ func TestRunnerRunStructuredRepairRemainsBoundedAndUsesEffectiveModelSettings(t
if repairer.reqs[0].Target.TimeoutSeconds != 22 { if repairer.reqs[0].Target.TimeoutSeconds != 22 {
t.Fatalf("expected repair to use effective timeout, got %d", repairer.reqs[0].Target.TimeoutSeconds) t.Fatalf("expected repair to use effective timeout, got %d", repairer.reqs[0].Target.TimeoutSeconds)
} }
if llmClient.lastReq.Target.BackendID != "custom" ||
repairer.reqs[0].Target.BackendID != "custom" ||
res.SelectedBackendID != "custom" {
t.Fatalf("expected backend identity in generation, repair, and result: generate=%q repair=%q result=%q",
llmClient.lastReq.Target.BackendID, repairer.reqs[0].Target.BackendID, res.SelectedBackendID)
}
} }
func TestRunnerRunJSONSchemaRepairCarriesStructuredOutputSpec(t *testing.T) { func TestRunnerRunJSONSchemaRepairCarriesStructuredOutputSpec(t *testing.T) {
@@ -1546,13 +1574,13 @@ func TestRunnerRunJSONSchemaRepairCarriesStructuredOutputSpec(t *testing.T) {
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: `{"events":[1]}`}} llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: `{"events":[1]}`}}
runner := NewRunnerWithRepairer( runner := NewRunnerWithRepairer(
&fakePromptRepo{def: def}, &fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(), defaultArtifactReader(),
defaultRenderer(), defaultRenderer(),
llmClient, llmClient,
validator, validator,
repairer, repairer)
)
_, err := runner.Run(context.Background(), domain.RunRequest{ _, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p", PromptID: "p",
@@ -1629,13 +1657,12 @@ func TestResolveExecutionTargetProfileValuesPopulateAllSupportedFields(t *testin
ServiceTier: "priority", ServiceTier: "priority",
ReasoningEffort: "low", ReasoningEffort: "low",
APIKeyEnv: "PROFILE_KEY", APIKeyEnv: "PROFILE_KEY",
APIKeyRequired: true,
ExtraParams: map[string]any{ ExtraParams: map[string]any{
"profile_option": "enabled", "profile_option": "enabled",
}, },
} }
target, presence, err := resolveExecutionTarget(profileValue, nil) target, presence, err := resolveExecutionTarget(nil, profileValue, nil)
if err != nil { if err != nil {
t.Fatalf("expected no error, got %v", err) t.Fatalf("expected no error, got %v", err)
} }
@@ -1690,7 +1717,7 @@ func TestResolveExecutionTargetRuntimeOverridesBeatProfileForAllOverrideableFiel
}, },
} }
target, presence, err := resolveExecutionTarget(profileValue, override) target, presence, err := resolveExecutionTarget(nil, profileValue, override)
if err != nil { if err != nil {
t.Fatalf("expected no error, got %v", err) t.Fatalf("expected no error, got %v", err)
} }
@@ -1813,6 +1840,118 @@ func defaultExecutionProfile() *domain.ExecutionProfile {
} }
} }
func TestResolveExecutionTargetUsesBackendProfileAndRequestPrecedence(t *testing.T) {
backendValue := &domain.Backend{
ID: "custom",
Endpoint: "http://backend/v1",
APIKeyEnv: "BACKEND_KEY",
ExtraParams: map[string]any{"backend": true},
}
profileValue := &domain.ExecutionProfile{
ID: "exec",
BackendID: "custom",
Endpoint: "http://profile/v1",
Model: "profile-model",
APIKeyEnv: "PROFILE_KEY",
ExtraParams: map[string]any{"profile": true},
}
override := &domain.ExecutionTargetOverride{
Endpoint: "http://request/v1",
APIKeyEnv: "REQUEST_KEY",
ExtraParams: map[string]any{"request": true},
}
target, _, err := resolveExecutionTarget(backendValue, profileValue, override)
if err != nil {
t.Fatalf("resolve target: %v", err)
}
if target.BackendID != "custom" {
t.Fatalf("endpoint override changed backend identity: %+v", target)
}
if target.Endpoint != "http://request/v1" || target.APIKeyEnv != "REQUEST_KEY" {
t.Fatalf("request values did not win: %+v", target)
}
if !reflect.DeepEqual(target.ExtraParams, map[string]any{"request": true}) {
t.Fatalf("expected whole-map request replacement, got %#v", target.ExtraParams)
}
target, _, err = resolveExecutionTarget(backendValue, &domain.ExecutionProfile{
ID: "exec", BackendID: "custom", Model: "profile-model",
}, nil)
if err != nil {
t.Fatalf("resolve backend defaults: %v", err)
}
if target.Endpoint != backendValue.Endpoint ||
target.APIKeyEnv != backendValue.APIKeyEnv ||
!reflect.DeepEqual(target.ExtraParams, backendValue.ExtraParams) {
t.Fatalf("backend defaults were not inherited: %+v", target)
}
}
func TestRunnerPrepareBackendResolutionAndCredentialPrecedence(t *testing.T) {
resolver := fakeBackendResolver{backends: map[string]domain.Backend{
"custom": {ID: "custom", Endpoint: "http://backend/v1", APIKeyEnv: "BACKEND_KEY"},
}}
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
t.Run("unknown backend is a profile load failure", func(t *testing.T) {
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", BackendID: "unknown", Model: "model"},
}}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if !errors.Is(err, ErrProfileLoad) {
t.Fatalf("expected ErrProfileLoad, got %v", err)
}
})
t.Run("nil resolver is a profile load failure", func(t *testing.T) {
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", BackendID: "custom", Model: "model"},
}}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if !errors.Is(err, ErrProfileLoad) {
t.Fatalf("expected ErrProfileLoad, got %v", err)
}
})
t.Run("request environment wins", func(t *testing.T) {
t.Setenv("REQUEST_KEY", "request-secret")
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", BackendID: "custom", Model: "model", APIKeyEnv: "PROFILE_KEY"},
}}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(),
Execution: &domain.ExecutionTargetOverride{APIKeyEnv: "REQUEST_KEY"},
})
if err != nil {
t.Fatalf("prepare: %v", err)
}
if prepared.EffectiveModelParams.APIKeyEnv != "REQUEST_KEY" {
t.Fatalf("unexpected credential source: %+v", prepared.EffectiveModelParams)
}
})
t.Run("required direct key clears inherited environment", func(t *testing.T) {
t.Setenv("BACKEND_KEY", "backend-secret")
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", BackendID: "custom", Model: "model", APIKeyRequired: true},
}}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if !errors.Is(err, ErrAPIKeyRequired) {
t.Fatalf("expected ErrAPIKeyRequired, got %v", err)
}
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(), APIKey: "direct-secret",
})
if err != nil {
t.Fatalf("prepare with direct key: %v", err)
}
if prepared.EffectiveModelParams.APIKeyEnv != "" {
t.Fatalf("expected inherited environment name to be cleared, got %q", prepared.EffectiveModelParams.APIKeyEnv)
}
})
}
func defaultArtifactReader() *fakeArtifactReader { func defaultArtifactReader() *fakeArtifactReader {
return &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{ return &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{
"a://ok": {Body: []byte("x"), Hash: hashString("x")}, "a://ok": {Body: []byte("x"), Hash: hashString("x")},
@@ -1838,10 +1977,11 @@ func intPtr(v int) *int {
func newMinimalRunner(promptRepo *fakePromptRepo, execRepo *fakeExecutionProfileRepo) *Runner { func newMinimalRunner(promptRepo *fakePromptRepo, execRepo *fakeExecutionProfileRepo) *Runner {
return NewRunner( return NewRunner(
promptRepo, promptRepo,
execRepo, execRepo, nil,
defaultArtifactReader(), defaultArtifactReader(),
defaultRenderer(), defaultRenderer(),
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
nil, nil)
)
} }

View File

@@ -26,6 +26,7 @@ func (r PreparedRun) MarshalJSON() ([]byte, error) {
PromptVersion string `json:"prompt_version,omitempty"` PromptVersion string `json:"prompt_version,omitempty"`
PromptHash string `json:"prompt_hash,omitempty"` PromptHash string `json:"prompt_hash,omitempty"`
SelectedProfileID string `json:"selected_profile_id"` SelectedProfileID string `json:"selected_profile_id"`
SelectedBackendID string `json:"selected_backend_id,omitempty"`
EffectiveModelParams ExecutionTarget `json:"effective_model_params"` EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
OutputContract OutputContract `json:"output_contract"` OutputContract OutputContract `json:"output_contract"`
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"` StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
@@ -41,6 +42,7 @@ func (r PreparedRun) MarshalJSON() ([]byte, error) {
PromptVersion: r.PromptVersion, PromptVersion: r.PromptVersion,
PromptHash: r.PromptHash, PromptHash: r.PromptHash,
SelectedProfileID: r.SelectedProfileID, SelectedProfileID: r.SelectedProfileID,
SelectedBackendID: r.SelectedBackendID,
EffectiveModelParams: r.EffectiveModelParams, EffectiveModelParams: r.EffectiveModelParams,
OutputContract: r.OutputContract, OutputContract: r.OutputContract,
StructuredOutput: r.StructuredOutput, StructuredOutput: r.StructuredOutput,
@@ -81,6 +83,7 @@ func (r RunResult) MarshalJSON() ([]byte, error) {
PromptHash: r.PromptHash, PromptHash: r.PromptHash,
RenderedPromptHash: r.RenderedPromptHash, RenderedPromptHash: r.RenderedPromptHash,
SelectedProfileID: r.SelectedProfileID, SelectedProfileID: r.SelectedProfileID,
SelectedBackendID: r.SelectedBackendID,
ModelName: r.ModelName, ModelName: r.ModelName,
Endpoint: r.Endpoint, Endpoint: r.Endpoint,
EffectiveModelParams: r.EffectiveModelParams, EffectiveModelParams: r.EffectiveModelParams,
@@ -110,6 +113,7 @@ func (r *RunResult) UnmarshalJSON(data []byte) error {
PromptHash: wire.PromptHash, PromptHash: wire.PromptHash,
RenderedPromptHash: wire.RenderedPromptHash, RenderedPromptHash: wire.RenderedPromptHash,
SelectedProfileID: wire.SelectedProfileID, SelectedProfileID: wire.SelectedProfileID,
SelectedBackendID: wire.SelectedBackendID,
ModelName: wire.ModelName, ModelName: wire.ModelName,
Endpoint: wire.Endpoint, Endpoint: wire.Endpoint,
EffectiveModelParams: wire.EffectiveModelParams, EffectiveModelParams: wire.EffectiveModelParams,
@@ -136,6 +140,7 @@ type runResultJSON struct {
PromptHash string `json:"prompt_hash,omitempty"` PromptHash string `json:"prompt_hash,omitempty"`
RenderedPromptHash string `json:"rendered_prompt_hash"` RenderedPromptHash string `json:"rendered_prompt_hash"`
SelectedProfileID string `json:"selected_profile_id"` SelectedProfileID string `json:"selected_profile_id"`
SelectedBackendID string `json:"selected_backend_id,omitempty"`
ModelName string `json:"model_name"` ModelName string `json:"model_name"`
Endpoint string `json:"endpoint"` Endpoint string `json:"endpoint"`
EffectiveModelParams ExecutionTarget `json:"effective_model_params"` EffectiveModelParams ExecutionTarget `json:"effective_model_params"`

View File

@@ -6,16 +6,21 @@ import (
"fmt" "fmt"
"strings" "strings"
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/domain" "gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/profile" "gitea.maximumdirect.net/eric/promptkit/internal/profile"
) )
// BackendOpenRouter identifies Promptkit's built-in OpenRouter backend.
const BackendOpenRouter = backend.OpenRouterID
// OpenAICompatibleProfile returns an ordinary in-memory Profile for an // OpenAICompatibleProfile returns an ordinary in-memory Profile for an
// OpenAI-compatible chat-completions endpoint. // OpenAI-compatible chat-completions endpoint.
// //
// It does not register global state, maintain a model catalog, or resolve // It does not register global state, maintain a model catalog, or resolve
// credentials. If APIKeyRequired is true, callers satisfy it with // credentials. If APIKeyRequired is true, callers satisfy it with
// RunRequest.APIKey. Raw API keys do not belong in profiles. // RunRequest.APIKey or an explicit request ExecutionTargetOverride.APIKeyEnv.
// Raw API keys do not belong in profiles.
// //
// The function copies the ExtraParams map itself but does not recursively copy // The function copies the ExtraParams map itself but does not recursively copy
// nested values. Validation and a deep copy occur when NewEngine applies a // nested values. Validation and a deep copy occur when NewEngine applies a
@@ -23,6 +28,7 @@ import (
func OpenAICompatibleProfile(cfg OpenAICompatibleProfileConfig) Profile { func OpenAICompatibleProfile(cfg OpenAICompatibleProfileConfig) Profile {
return Profile{ return Profile{
ID: cfg.ID, ID: cfg.ID,
BackendID: cfg.BackendID,
Endpoint: cfg.Endpoint, Endpoint: cfg.Endpoint,
Model: cfg.Model, Model: cfg.Model,
Temperature: cfg.Temperature, Temperature: cfg.Temperature,
@@ -85,6 +91,7 @@ func toDomainProfile(publicProfile Profile) (domain.ExecutionProfile, error) {
} }
prof := domain.ExecutionProfile{ prof := domain.ExecutionProfile{
ID: strings.TrimSpace(publicProfile.ID), ID: strings.TrimSpace(publicProfile.ID),
BackendID: strings.TrimSpace(publicProfile.BackendID),
Endpoint: publicProfile.Endpoint, Endpoint: publicProfile.Endpoint,
Model: publicProfile.Model, Model: publicProfile.Model,
Temperature: publicProfile.Temperature, Temperature: publicProfile.Temperature,
@@ -106,8 +113,8 @@ func validatePublicProfile(prof domain.ExecutionProfile) error {
if strings.TrimSpace(prof.ID) == "" { if strings.TrimSpace(prof.ID) == "" {
return errors.New("id is required") return errors.New("id is required")
} }
if strings.TrimSpace(prof.Endpoint) == "" { if strings.TrimSpace(prof.BackendID) == "" && strings.TrimSpace(prof.Endpoint) == "" {
return errors.New("endpoint is required") return errors.New("backend or endpoint is required")
} }
if strings.TrimSpace(prof.Model) == "" { if strings.TrimSpace(prof.Model) == "" {
return errors.New("model is required") return errors.New("model is required")

View File

@@ -3,6 +3,7 @@ package promptkit_test
import ( import (
"context" "context"
"encoding/json" "encoding/json"
"errors"
"fmt" "fmt"
"strings" "strings"
"sync" "sync"
@@ -26,6 +27,62 @@ func TestPreparedRunJSONOmitsZeroTimingValues(t *testing.T) {
} }
} }
func TestBackendIdentityJSONNamesAndOmission(t *testing.T) {
values := []struct {
name string
value any
field string
}{
{name: "execution target", value: promptkit.ExecutionTarget{BackendID: promptkit.BackendOpenRouter}, field: "backend_id"},
{name: "prepared run", value: promptkit.PreparedRun{SelectedBackendID: promptkit.BackendOpenRouter}, field: "selected_backend_id"},
{name: "run result", value: promptkit.RunResult{SelectedBackendID: promptkit.BackendOpenRouter}, field: "selected_backend_id"},
}
for _, tt := range values {
t.Run(tt.name, func(t *testing.T) {
payload, err := json.Marshal(tt.value)
if err != nil {
t.Fatalf("marshal populated value: %v", err)
}
var object map[string]any
if err := json.Unmarshal(payload, &object); err != nil {
t.Fatalf("decode populated value: %v", err)
}
if object[tt.field] != promptkit.BackendOpenRouter {
t.Fatalf("expected %s=%q, got %s", tt.field, promptkit.BackendOpenRouter, payload)
}
})
}
emptyValues := []any{promptkit.ExecutionTarget{}, promptkit.PreparedRun{}, promptkit.RunResult{}}
for _, value := range emptyValues {
payload, err := json.Marshal(value)
if err != nil {
t.Fatalf("marshal empty value: %v", err)
}
if strings.Contains(string(payload), `"backend_id"`) || strings.Contains(string(payload), `"selected_backend_id"`) {
t.Fatalf("empty backend identity was not omitted: %s", payload)
}
}
}
func TestUnknownProfileBackendHasProfileLoadIdentity(t *testing.T) {
engine, err := promptkit.NewEngine(promptkit.Config{},
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
promptkit.WithProfiles(promptkit.Profile{ID: "profile", BackendID: "unknown", Model: "model"}),
)
if err != nil {
t.Fatalf("construct engine: %v", err)
}
_, err = engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
if !errors.Is(err, promptkit.ErrProfileLoad) {
t.Fatalf("expected ErrProfileLoad, got %v", err)
}
if errors.Is(err, promptkit.ErrInvalidRequest) {
t.Fatalf("unknown backend should not have invalid-request identity: %v", err)
}
}
func TestPreparedRunJSONTimingRoundTrips(t *testing.T) { func TestPreparedRunJSONTimingRoundTrips(t *testing.T) {
start := time.Date(2026, time.July, 29, 12, 0, 0, 0, time.UTC) start := time.Date(2026, time.July, 29, 12, 0, 0, 0, time.UTC)
prepared := promptkit.PreparedRun{ prepared := promptkit.PreparedRun{

View File

@@ -126,8 +126,12 @@ type PreparedRun struct {
// SelectedProfileID is the explicit request profile or prompt default that // SelectedProfileID is the explicit request profile or prompt default that
// supplied execution settings. // supplied execution settings.
SelectedProfileID string `json:"selected_profile_id"` SelectedProfileID string `json:"selected_profile_id"`
// SelectedBackendID is the selected profile's normalized backend ID. It is
// empty for an endpoint-only profile.
SelectedBackendID string `json:"selected_backend_id,omitempty"`
// EffectiveModelParams contains framework defaults overlaid by the selected // EffectiveModelParams contains framework defaults overlaid by the selected
// profile and then request overrides. It excludes resolved API-key values. // backend, profile, and then request overrides. It excludes resolved API-key
// values.
EffectiveModelParams ExecutionTarget `json:"effective_model_params"` EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
// OutputContract is the complete effective output contract. // OutputContract is the complete effective output contract.
OutputContract OutputContract `json:"output_contract"` OutputContract OutputContract `json:"output_contract"`
@@ -180,6 +184,9 @@ type RunResult struct {
RenderedPromptHash string `json:"rendered_prompt_hash"` RenderedPromptHash string `json:"rendered_prompt_hash"`
// SelectedProfileID identifies the profile used for execution. // SelectedProfileID identifies the profile used for execution.
SelectedProfileID string `json:"selected_profile_id"` SelectedProfileID string `json:"selected_profile_id"`
// SelectedBackendID is the selected profile's normalized backend ID. It is
// empty for an endpoint-only profile.
SelectedBackendID string `json:"selected_backend_id,omitempty"`
// ModelName is the effective model name and equals // ModelName is the effective model name and equals
// EffectiveModelParams.Model. // EffectiveModelParams.Model.
ModelName string `json:"model_name"` ModelName string `json:"model_name"`
@@ -259,6 +266,10 @@ type ArtifactReader interface {
// ExecutionTarget represents effective model runtime settings and has a stable // ExecutionTarget represents effective model runtime settings and has a stable
// JSON representation. It never exposes a resolved API-key value. // JSON representation. It never exposes a resolved API-key value.
type ExecutionTarget struct { type ExecutionTarget struct {
// BackendID is the effective routing identity selected by the profile. It
// remains unchanged when a profile or request overrides Endpoint and is
// empty for endpoint-only profiles.
BackendID string `json:"backend_id,omitempty"`
// Endpoint is the model-provider base URL. // Endpoint is the model-provider base URL.
Endpoint string `json:"endpoint"` Endpoint string `json:"endpoint"`
// Model is the provider model identifier. // Model is the provider model identifier.
@@ -323,9 +334,10 @@ type ExecutionTargetOverride struct {
// Profile is an in-memory execution profile for library consumers. // Profile is an in-memory execution profile for library consumers.
// //
// It is equivalent to a loaded profile file after validation. Raw API keys do // It is equivalent to a loaded profile file after validation. Raw API keys do
// not belong in profiles; use APIKeyRequired to require callers to provide // not belong in profiles; use APIKeyRequired to require callers to provide a
// RunRequest.APIKey for each request, or use profile YAML api_key_env with file // RunRequest.APIKey or explicit request ExecutionTargetOverride.APIKeyEnv, or
// and FS profile sources. Profile has no stable JSON representation. // use profile YAML api_key_env with file and FS profile sources. Profile has no
// stable JSON representation.
// //
// WithProfiles validates and copies Profile values during NewEngine. Numeric // WithProfiles validates and copies Profile values during NewEngine. Numeric
// zero, blank strings, and an empty ExtraParams map inherit framework defaults; // zero, blank strings, and an empty ExtraParams map inherit framework defaults;
@@ -333,7 +345,12 @@ type ExecutionTargetOverride struct {
type Profile struct { type Profile struct {
// ID is the required non-blank profile identifier. WithProfiles trims it. // ID is the required non-blank profile identifier. WithProfiles trims it.
ID string ID string
// Endpoint is the required non-blank model-provider base URL. // BackendID optionally selects an engine backend. WithProfiles trims it.
// Backend membership is checked when a request selects the profile.
BackendID string
// Endpoint is the model-provider base URL. It is required only when
// BackendID is blank and otherwise overrides the backend endpoint when
// non-blank.
Endpoint string Endpoint string
// Model is the required non-blank provider model identifier. // Model is the required non-blank provider model identifier.
Model string Model string
@@ -351,8 +368,9 @@ type Profile struct {
// ReasoningEffort is optional; a blank value inherits the framework // ReasoningEffort is optional; a blank value inherits the framework
// default. // default.
ReasoningEffort string ReasoningEffort string
// APIKeyRequired requires a non-blank RunRequest.APIKey. It does not store a // APIKeyRequired clears a backend's inherited API-key environment name and
// credential or enable environment lookup. // requires a non-blank RunRequest.APIKey unless the request explicitly
// supplies ExecutionTargetOverride.APIKeyEnv. It does not store a credential.
APIKeyRequired bool APIKeyRequired bool
// ExtraParams contains provider-specific JSON-compatible values. An empty // ExtraParams contains provider-specific JSON-compatible values. An empty
// map inherits framework defaults. WithProfiles validates and deeply copies // map inherits framework defaults. WithProfiles validates and deeply copies
@@ -364,13 +382,15 @@ type Profile struct {
// profile. // profile.
// //
// It contains ordinary profile fields for OpenAI-compatible chat-completions // It contains ordinary profile fields for OpenAI-compatible chat-completions
// endpoints. APIKeyRequired is satisfied by RunRequest.APIKey. Raw API keys do // endpoints. APIKeyRequired follows Profile.APIKeyRequired. Raw API keys do not
// not belong in this config. OpenAICompatibleProfileConfig has no stable JSON // belong in this config. OpenAICompatibleProfileConfig has no stable JSON
// representation and is not validated until its resulting Profile is supplied // representation and is not validated until its resulting Profile is supplied
// through WithProfiles to NewEngine. // through WithProfiles to NewEngine.
type OpenAICompatibleProfileConfig struct { type OpenAICompatibleProfileConfig struct {
// ID becomes Profile.ID. // ID becomes Profile.ID.
ID string ID string
// BackendID becomes Profile.BackendID.
BackendID string
// Endpoint becomes Profile.Endpoint. // Endpoint becomes Profile.Endpoint.
Endpoint string Endpoint string
// Model becomes Profile.Model. // Model becomes Profile.Model.