Resolve profiles through built-in backends
This commit is contained in:
@@ -32,6 +32,7 @@ func fromDomainPreparedRun(prepared *domain.PreparedRun) *PreparedRun {
|
|||||||
PromptVersion: prepared.PromptVersion,
|
PromptVersion: prepared.PromptVersion,
|
||||||
PromptHash: prepared.PromptHash,
|
PromptHash: prepared.PromptHash,
|
||||||
SelectedProfileID: prepared.SelectedProfileID,
|
SelectedProfileID: prepared.SelectedProfileID,
|
||||||
|
SelectedBackendID: prepared.SelectedBackendID,
|
||||||
EffectiveModelParams: fromDomainExecutionTarget(prepared.EffectiveModelParams),
|
EffectiveModelParams: fromDomainExecutionTarget(prepared.EffectiveModelParams),
|
||||||
OutputContract: fromDomainOutputContract(prepared.OutputContract),
|
OutputContract: fromDomainOutputContract(prepared.OutputContract),
|
||||||
StructuredOutput: fromDomainStructuredOutputSpec(prepared.StructuredOutput),
|
StructuredOutput: fromDomainStructuredOutputSpec(prepared.StructuredOutput),
|
||||||
@@ -59,6 +60,7 @@ func fromDomainRunResult(result *domain.RunResult) *RunResult {
|
|||||||
PromptHash: result.PromptHash,
|
PromptHash: result.PromptHash,
|
||||||
RenderedPromptHash: result.RenderedPromptHash,
|
RenderedPromptHash: result.RenderedPromptHash,
|
||||||
SelectedProfileID: result.SelectedProfileID,
|
SelectedProfileID: result.SelectedProfileID,
|
||||||
|
SelectedBackendID: result.SelectedBackendID,
|
||||||
ModelName: result.ModelName,
|
ModelName: result.ModelName,
|
||||||
Endpoint: result.Endpoint,
|
Endpoint: result.Endpoint,
|
||||||
EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams),
|
EffectiveModelParams: fromDomainExecutionTarget(result.EffectiveModelParams),
|
||||||
@@ -151,6 +153,7 @@ func toDomainExecutionTargetOverride(override *ExecutionTargetOverride) (*domain
|
|||||||
|
|
||||||
func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
|
func fromDomainExecutionTarget(target domain.ExecutionTarget) ExecutionTarget {
|
||||||
return ExecutionTarget{
|
return ExecutionTarget{
|
||||||
|
BackendID: target.BackendID,
|
||||||
Endpoint: target.Endpoint,
|
Endpoint: target.Endpoint,
|
||||||
Model: target.Model,
|
Model: target.Model,
|
||||||
Temperature: target.Temperature,
|
Temperature: target.Temperature,
|
||||||
|
|||||||
@@ -136,7 +136,7 @@ A profile supplies model execution settings:
|
|||||||
|
|
||||||
```yaml
|
```yaml
|
||||||
id: local-summary
|
id: local-summary
|
||||||
endpoint: http://localhost:8000/v1
|
backend: openrouter
|
||||||
model: example-model
|
model: example-model
|
||||||
temperature: 0.2
|
temperature: 0.2
|
||||||
max_tokens: 500
|
max_tokens: 500
|
||||||
@@ -144,7 +144,6 @@ top_p: 0.95
|
|||||||
timeout_seconds: 90
|
timeout_seconds: 90
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
reasoning_effort: medium
|
reasoning_effort: medium
|
||||||
api_key_env: EXAMPLE_API_KEY
|
|
||||||
extra_params:
|
extra_params:
|
||||||
provider_option: enabled
|
provider_option: enabled
|
||||||
```
|
```
|
||||||
@@ -152,7 +151,8 @@ extra_params:
|
|||||||
| Field | Required | Meaning |
|
| Field | Required | Meaning |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
| `id` | yes | Non-empty profile identifier. IDs must be unique within one source. |
|
| `id` | yes | Non-empty profile identifier. IDs must be unique within one source. |
|
||||||
| `endpoint` | yes | Non-empty OpenAI-compatible base URL, including an API version path when required. |
|
| `backend` | unless `endpoint` is present | Backend registry ID. It is trimmed and registry membership is checked when the profile is prepared. |
|
||||||
|
| `endpoint` | unless `backend` is present | Non-empty OpenAI-compatible base URL, including an API version path when required. When both connection fields are present, this overrides the backend endpoint without changing backend identity. |
|
||||||
| `model` | yes | Non-empty provider model name. |
|
| `model` | yes | Non-empty provider model name. |
|
||||||
| `temperature` | no | Number from 0 through 2. |
|
| `temperature` | no | Number from 0 through 2. |
|
||||||
| `max_tokens` | no | Integer zero or greater. |
|
| `max_tokens` | no | Integer zero or greater. |
|
||||||
@@ -166,6 +166,9 @@ extra_params:
|
|||||||
Raw `api_key` is prohibited in profile YAML. Store only an environment
|
Raw `api_key` is prohibited in profile YAML. Store only an environment
|
||||||
variable name in `api_key_env`.
|
variable name in `api_key_env`.
|
||||||
|
|
||||||
|
Promptkit does not infer a backend from a model or endpoint. Endpoint-only
|
||||||
|
profiles remain supported and have no effective backend ID.
|
||||||
|
|
||||||
`extra_params` accepts null, booleans, finite numbers, strings, arrays, and
|
`extra_params` accepts null, booleans, finite numbers, strings, arrays, and
|
||||||
objects with string keys. Keys must be non-empty. With the built-in client,
|
objects with string keys. Keys must be non-empty. With the built-in client,
|
||||||
they also cannot collide with the standard fields listed in the
|
they also cannot collide with the standard fields listed in the
|
||||||
@@ -176,8 +179,9 @@ they also cannot collide with the standard fields listed in the
|
|||||||
Execution settings resolve in this order:
|
Execution settings resolve in this order:
|
||||||
|
|
||||||
1. framework defaults;
|
1. framework defaults;
|
||||||
2. the selected profile; and
|
2. the selected backend, when the profile names one;
|
||||||
3. request `ExecutionTargetOverride` values.
|
3. the selected profile; and
|
||||||
|
4. request `ExecutionTargetOverride` values.
|
||||||
|
|
||||||
The framework defaults are:
|
The framework defaults are:
|
||||||
|
|
||||||
@@ -194,8 +198,10 @@ explicit zero is preserved. In particular, an explicit request
|
|||||||
`timeout_seconds` of zero disables the per-generation deadline while leaving
|
`timeout_seconds` of zero disables the per-generation deadline while leaving
|
||||||
the caller context and transport timeout intact.
|
the caller context and transport timeout intact.
|
||||||
|
|
||||||
Non-empty request strings replace profile strings. A non-empty request
|
Non-empty profile strings replace backend defaults, and non-empty request
|
||||||
`ExtraParams` map replaces the profile map rather than merging keys.
|
strings replace both. Backend identity is retained when either layer overrides
|
||||||
|
the endpoint. A non-empty `extra_params` map at each layer replaces the entire
|
||||||
|
lower-precedence map rather than merging keys.
|
||||||
The [outbound integration contract](integrations/openai-compatible-chat.md)
|
The [outbound integration contract](integrations/openai-compatible-chat.md)
|
||||||
defines how the effective settings are serialized.
|
defines how the effective settings are serialized.
|
||||||
|
|
||||||
@@ -217,9 +223,11 @@ invalid matching profile is an error and does not fall back. In-memory
|
|||||||
|
|
||||||
## Built-In Profile Catalog
|
## Built-In Profile Catalog
|
||||||
|
|
||||||
Built-ins use the OpenRouter-compatible endpoint and
|
Every built-in selects the `openrouter` backend. The engine's built-in backend
|
||||||
`OPENROUTER_API_KEY`. A custom or in-memory profile with the same ID takes
|
registry supplies `https://openrouter.ai/api/v1` and the environment-variable
|
||||||
precedence.
|
name `OPENROUTER_API_KEY`, so individual profiles contain only model and
|
||||||
|
generation settings. A custom or in-memory profile with the same profile ID
|
||||||
|
takes precedence.
|
||||||
|
|
||||||
| Provider | ID | Model |
|
| Provider | ID | Model |
|
||||||
| --- | --- | --- |
|
| --- | --- | --- |
|
||||||
@@ -270,6 +278,10 @@ prompt, profile, schema, or example files:
|
|||||||
- a request can provide a direct `APIKey` or override `APIKeyEnv`; and
|
- a request can provide a direct `APIKey` or override `APIKeyEnv`; and
|
||||||
- a direct request key takes precedence over environment lookup.
|
- a direct request key takes precedence over environment lookup.
|
||||||
|
|
||||||
|
After a direct request key, the credential-source precedence is request
|
||||||
|
`APIKeyEnv`, profile `api_key_env`, then the backend default. An in-memory
|
||||||
|
profile with `APIKeyRequired` clears an inherited backend environment name and
|
||||||
|
requires a direct key unless the request explicitly supplies `APIKeyEnv`.
|
||||||
Promptkit validates required credential availability during preparation.
|
Promptkit validates required credential availability during preparation.
|
||||||
Direct keys are excluded from JSON results and redacted by public string
|
Direct keys are excluded from JSON results and redacted by public string
|
||||||
formatters. Environment-variable names may appear in prepared metadata, but
|
formatters. Environment-variable names may appear in prepared metadata, but
|
||||||
|
|||||||
@@ -17,10 +17,11 @@ and override semantics consumed by the runner.
|
|||||||
## Collaborators
|
## Collaborators
|
||||||
|
|
||||||
`Runner` coordinates narrow internal interfaces for prompt definitions,
|
`Runner` coordinates narrow internal interfaces for prompt definitions,
|
||||||
profiles, artifacts, rendering, model generation, and validation. Schema
|
profiles, backend resolution, artifacts, rendering, model generation, and
|
||||||
documents are loaded through the validator's optional schema-loader interface.
|
validation. The root engine supplies an immutable built-in backend registry.
|
||||||
An output repairer can be injected internally, but the ordinary runner
|
Schema documents are loaded through the validator's optional schema-loader
|
||||||
constructor does not enable one.
|
interface. An output repairer can be injected internally, but the ordinary
|
||||||
|
runner constructor does not enable one.
|
||||||
|
|
||||||
Each invocation carries its state in request, prepared-run, and result values.
|
Each invocation carries its state in request, prepared-run, and result values.
|
||||||
The runner has no durable run or session store.
|
The runner has no durable run or session store.
|
||||||
@@ -32,20 +33,25 @@ The runner has no durable run or session store.
|
|||||||
1. validate the prompt selection and load the prompt definition;
|
1. validate the prompt selection and load the prompt definition;
|
||||||
2. hash the loaded definition;
|
2. hash the loaded definition;
|
||||||
3. select the request profile or the prompt's default profile;
|
3. select the request profile or the prompt's default profile;
|
||||||
4. resolve application-neutral defaults, profile values, and explicit request
|
4. resolve the profile's backend ID, when present;
|
||||||
overrides in that order;
|
5. resolve application-neutral defaults, backend defaults, profile values,
|
||||||
5. validate endpoint, model, numeric overrides, and credential requirements;
|
and explicit request overrides in that order;
|
||||||
6. resolve the output contract and load a structured-output schema when
|
6. validate endpoint, model, numeric overrides, and credential requirements;
|
||||||
|
7. resolve the output contract and load a structured-output schema when
|
||||||
required;
|
required;
|
||||||
7. load and hash input artifacts;
|
8. load and hash input artifacts;
|
||||||
8. render and hash the prompt; and
|
9. render and hash the prompt; and
|
||||||
9. return the effective settings, source identities, messages, hashes, and
|
10. return the effective settings, source identities, messages, hashes, and
|
||||||
preparation timing.
|
preparation timing.
|
||||||
|
|
||||||
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
|
Pointer-based numeric overrides preserve an explicit zero. Invalid negative or
|
||||||
out-of-range values fail as invalid requests. A direct API key takes
|
out-of-range values fail as invalid requests. Endpoint overrides do not change
|
||||||
precedence over environment lookup for execution; secret values remain
|
the selected backend identity. Non-empty extra-parameter maps replace whole
|
||||||
excluded from serialized metadata.
|
lower-precedence maps. A direct API key takes precedence over environment
|
||||||
|
lookup; otherwise request, profile, and backend environment-variable names
|
||||||
|
apply in that order. A profile requiring a direct key clears an inherited
|
||||||
|
backend environment name unless the request supplies its own name. Secret
|
||||||
|
values remain excluded from serialized metadata.
|
||||||
|
|
||||||
## Run Flow
|
## Run Flow
|
||||||
|
|
||||||
@@ -60,8 +66,10 @@ target, validation errors, prior output, and structured-output specification.
|
|||||||
This capability remains internal and is not a public option.
|
This capability remains internal and is not a public option.
|
||||||
|
|
||||||
A successful result includes the output artifact and raw output, validation
|
A successful result includes the output artifact and raw output, validation
|
||||||
state, prompt and rendered-prompt hashes, selected profile, effective settings,
|
state, prompt and rendered-prompt hashes, selected profile and backend,
|
||||||
input hashes, token usage, a generated run identifier, and UTC timing.
|
effective settings, input hashes, token usage, a generated run identifier, and
|
||||||
|
UTC timing. The same effective target, including backend identity, reaches
|
||||||
|
generation and any repair attempt.
|
||||||
|
|
||||||
## Failure Categories
|
## Failure Categories
|
||||||
|
|
||||||
@@ -71,13 +79,16 @@ validation failures. Wrapping preserves the package identities mapped by the
|
|||||||
public facade and retains collaborator identities where they are part of the
|
public facade and retains collaborator identities where they are part of the
|
||||||
internal contract. Context cancellation propagates through the invoked
|
internal contract. Context cancellation propagates through the invoked
|
||||||
collaborator and is classified by the owning operation.
|
collaborator and is classified by the owning operation.
|
||||||
|
An unknown selected backend, or a selected backend with no configured resolver,
|
||||||
|
is classified as a profile-load failure.
|
||||||
|
|
||||||
## Test Ownership And Changes
|
## Test Ownership And Changes
|
||||||
|
|
||||||
The [runner tests](../../internal/usecase/runner_test.go) own preparation order,
|
The [runner tests](../../internal/usecase/runner_test.go) own preparation order,
|
||||||
selection and override precedence, schema-before-generation behavior, hashing,
|
selection and override precedence, schema-before-generation behavior, hashing,
|
||||||
generation and validation outcomes, bounded repair, credentials and redaction,
|
generation and validation outcomes, backend propagation, bounded repair,
|
||||||
error categories, artifact metadata, usage, and timing.
|
credentials and redaction, error categories, artifact metadata, usage, and
|
||||||
|
timing.
|
||||||
|
|
||||||
Changes to orchestration should continue to use the existing package
|
Changes to orchestration should continue to use the existing package
|
||||||
interfaces, keep request state local to an invocation, and preserve `Run`'s use
|
interfaces, keep request state local to an invocation, and preserve `Run`'s use
|
||||||
|
|||||||
@@ -24,13 +24,19 @@ duplicate detection, and source containment:
|
|||||||
|
|
||||||
`internal/profile` loads and validates execution profiles from an
|
`internal/profile` loads and validates execution profiles from an
|
||||||
operating-system filesystem or an `fs.FS`. It supports a primary repository
|
operating-system filesystem or an `fs.FS`. It supports a primary repository
|
||||||
with fallback only when the primary reports that a profile is absent.
|
with fallback only when the primary reports that a profile is absent. Strict
|
||||||
|
YAML decoding recognizes the optional `backend` field, trims its value, and
|
||||||
|
requires a model plus at least one non-blank backend or endpoint. Loading does
|
||||||
|
not check registry membership because the available registry belongs to the
|
||||||
|
assembled engine; the runner checks membership during preparation.
|
||||||
|
|
||||||
`internal/profile/builtin` embeds the maintained built-in profile catalog and
|
`internal/profile/builtin` embeds the maintained built-in profile catalog and
|
||||||
can place a caller-selected repository ahead of that catalog. Profile behavior
|
can place a caller-selected repository ahead of that catalog. Every embedded
|
||||||
is owned by the
|
profile selects `openrouter` and inherits its endpoint and credential
|
||||||
|
environment-variable name from the built-in backend registry rather than
|
||||||
|
repeating those values. Profile behavior is owned by the
|
||||||
[profile repository tests](../../internal/profile/repository_test.go), while
|
[profile repository tests](../../internal/profile/repository_test.go), while
|
||||||
catalog completeness, duplicate IDs, and overlay behavior are owned by the
|
catalog completeness, backend-selection invariant, duplicate IDs, and overlay behavior are owned by the
|
||||||
[built-in repository tests](../../internal/profile/builtin/repository_test.go).
|
[built-in repository tests](../../internal/profile/builtin/repository_test.go).
|
||||||
|
|
||||||
## Ordinary Artifacts
|
## Ordinary Artifacts
|
||||||
|
|||||||
@@ -299,6 +299,8 @@ immutable and the final repository race suite exercises concurrent reads.
|
|||||||
|
|
||||||
## Stage 2 — Profile Selection And Effective Resolution
|
## Stage 2 — Profile Selection And Effective Resolution
|
||||||
|
|
||||||
|
**Status:** Complete.
|
||||||
|
|
||||||
### Goal
|
### Goal
|
||||||
|
|
||||||
Make the built-in OpenRouter backend selectable by file and in-memory profiles,
|
Make the built-in OpenRouter backend selectable by file and in-memory profiles,
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ import (
|
|||||||
"time"
|
"time"
|
||||||
|
|
||||||
artifactadapter "gitea.maximumdirect.net/eric/promptkit/internal/artifact"
|
artifactadapter "gitea.maximumdirect.net/eric/promptkit/internal/artifact"
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
"gitea.maximumdirect.net/eric/promptkit/internal/defaults"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
"gitea.maximumdirect.net/eric/promptkit/internal/llm"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||||
@@ -335,6 +336,11 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
|||||||
profiles = profile.NewOverlayRepository(options.memoryProfiles, profiles)
|
profiles = profile.NewOverlayRepository(options.memoryProfiles, profiles)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
backendRegistry, err := backend.NewRegistry(nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("%w: failed to construct backend registry: %v", ErrInvalidConfig, err)
|
||||||
|
}
|
||||||
|
|
||||||
validator := options.validator
|
validator := options.validator
|
||||||
if !options.validatorSource {
|
if !options.validatorSource {
|
||||||
schemaDir := cfg.SchemaDir
|
schemaDir := cfg.SchemaDir
|
||||||
@@ -365,6 +371,7 @@ func NewEngine(cfg Config, opts ...Option) (*Engine, error) {
|
|||||||
runner: usecase.NewRunner(
|
runner: usecase.NewRunner(
|
||||||
promptDefs,
|
promptDefs,
|
||||||
profiles,
|
profiles,
|
||||||
|
backendRegistry,
|
||||||
artifacts,
|
artifacts,
|
||||||
prompt.NewGoRenderer(),
|
prompt.NewGoRenderer(),
|
||||||
llmClient,
|
llmClient,
|
||||||
|
|||||||
@@ -1271,6 +1271,18 @@ func TestPrepareUsesBuiltInProfileWithoutProfileDir(t *testing.T) {
|
|||||||
if prepared.SelectedProfileID != "mistral-small-3" {
|
if prepared.SelectedProfileID != "mistral-small-3" {
|
||||||
t.Fatalf("unexpected selected profile: %q", prepared.SelectedProfileID)
|
t.Fatalf("unexpected selected profile: %q", prepared.SelectedProfileID)
|
||||||
}
|
}
|
||||||
|
if prepared.SelectedBackendID != promptkit.BackendOpenRouter {
|
||||||
|
t.Fatalf("unexpected selected backend: %q", prepared.SelectedBackendID)
|
||||||
|
}
|
||||||
|
if prepared.EffectiveModelParams.BackendID != promptkit.BackendOpenRouter {
|
||||||
|
t.Fatalf("unexpected effective backend: %q", prepared.EffectiveModelParams.BackendID)
|
||||||
|
}
|
||||||
|
if prepared.EffectiveModelParams.Endpoint != "https://openrouter.ai/api/v1" {
|
||||||
|
t.Fatalf("unexpected built-in endpoint: %q", prepared.EffectiveModelParams.Endpoint)
|
||||||
|
}
|
||||||
|
if prepared.EffectiveModelParams.APIKeyEnv != "OPENROUTER_API_KEY" {
|
||||||
|
t.Fatalf("unexpected built-in api key environment name: %q", prepared.EffectiveModelParams.APIKeyEnv)
|
||||||
|
}
|
||||||
if prepared.EffectiveModelParams.Model != "mistralai/mistral-small-3.2-24b-instruct" {
|
if prepared.EffectiveModelParams.Model != "mistralai/mistral-small-3.2-24b-instruct" {
|
||||||
t.Fatalf("unexpected built-in model: %q", prepared.EffectiveModelParams.Model)
|
t.Fatalf("unexpected built-in model: %q", prepared.EffectiveModelParams.Model)
|
||||||
}
|
}
|
||||||
@@ -1641,6 +1653,7 @@ func TestOpenAICompatibleProfileRunsThroughNormalProfilePath(t *testing.T) {
|
|||||||
fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
|
fake := &fakeLLMClient{response: &promptkit.GenerateResponse{Content: "ok"}}
|
||||||
prof := promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
prof := promptkit.OpenAICompatibleProfile(promptkit.OpenAICompatibleProfileConfig{
|
||||||
ID: "template-profile",
|
ID: "template-profile",
|
||||||
|
BackendID: " openrouter ",
|
||||||
Endpoint: "http://template/v1",
|
Endpoint: "http://template/v1",
|
||||||
Model: "template-model",
|
Model: "template-model",
|
||||||
APIKeyRequired: true,
|
APIKeyRequired: true,
|
||||||
@@ -1672,7 +1685,9 @@ func TestOpenAICompatibleProfileRunsThroughNormalProfilePath(t *testing.T) {
|
|||||||
if len(fake.requests) != 1 {
|
if len(fake.requests) != 1 {
|
||||||
t.Fatalf("expected one request, got %d", len(fake.requests))
|
t.Fatalf("expected one request, got %d", len(fake.requests))
|
||||||
}
|
}
|
||||||
if fake.requests[0].Target.Model != "template-model" || fake.requests[0].APIKey != "template-key" {
|
if fake.requests[0].Target.BackendID != promptkit.BackendOpenRouter ||
|
||||||
|
fake.requests[0].Target.Model != "template-model" ||
|
||||||
|
fake.requests[0].APIKey != "template-key" {
|
||||||
t.Fatalf("unexpected generated request: %+v", fake.requests[0])
|
t.Fatalf("unexpected generated request: %+v", fake.requests[0])
|
||||||
}
|
}
|
||||||
if !reflect.DeepEqual(fake.requests[0].Target.ExtraParams, map[string]any{"provider": "template"}) {
|
if !reflect.DeepEqual(fake.requests[0].Target.ExtraParams, map[string]any{"provider": "template"}) {
|
||||||
|
|||||||
@@ -81,6 +81,7 @@ type RunResult struct {
|
|||||||
PromptHash string
|
PromptHash string
|
||||||
RenderedPromptHash string
|
RenderedPromptHash string
|
||||||
SelectedProfileID string
|
SelectedProfileID string
|
||||||
|
SelectedBackendID string
|
||||||
ModelName string
|
ModelName string
|
||||||
Endpoint string
|
Endpoint string
|
||||||
EffectiveModelParams ExecutionTarget
|
EffectiveModelParams ExecutionTarget
|
||||||
@@ -98,6 +99,7 @@ type PreparedRun struct {
|
|||||||
PromptVersion string `json:"prompt_version,omitempty"`
|
PromptVersion string `json:"prompt_version,omitempty"`
|
||||||
PromptHash string `json:"prompt_hash,omitempty"`
|
PromptHash string `json:"prompt_hash,omitempty"`
|
||||||
SelectedProfileID string `json:"selected_profile_id"`
|
SelectedProfileID string `json:"selected_profile_id"`
|
||||||
|
SelectedBackendID string `json:"selected_backend_id,omitempty"`
|
||||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||||
TargetPresence ExecutionTargetPresence `json:"-"`
|
TargetPresence ExecutionTargetPresence `json:"-"`
|
||||||
OutputContract OutputContract `json:"output_contract"`
|
OutputContract OutputContract `json:"output_contract"`
|
||||||
@@ -168,6 +170,7 @@ type Backend struct {
|
|||||||
// ExecutionProfile describes how and where to execute a model.
|
// ExecutionProfile describes how and where to execute a model.
|
||||||
type ExecutionProfile struct {
|
type ExecutionProfile struct {
|
||||||
ID string `yaml:"id"`
|
ID string `yaml:"id"`
|
||||||
|
BackendID string `yaml:"backend"`
|
||||||
Endpoint string `yaml:"endpoint"`
|
Endpoint string `yaml:"endpoint"`
|
||||||
Model string `yaml:"model"`
|
Model string `yaml:"model"`
|
||||||
Temperature float64 `yaml:"temperature"`
|
Temperature float64 `yaml:"temperature"`
|
||||||
@@ -206,6 +209,7 @@ type ExecutionTargetPresence struct {
|
|||||||
|
|
||||||
// ExecutionTarget represents effective model runtime settings for a run.
|
// ExecutionTarget represents effective model runtime settings for a run.
|
||||||
type ExecutionTarget struct {
|
type ExecutionTarget struct {
|
||||||
|
BackendID string `yaml:"backend" json:"backend_id,omitempty"`
|
||||||
Endpoint string `yaml:"endpoint" json:"endpoint"`
|
Endpoint string `yaml:"endpoint" json:"endpoint"`
|
||||||
Model string `yaml:"model" json:"model"`
|
Model string `yaml:"model" json:"model"`
|
||||||
Temperature float64 `yaml:"temperature" json:"temperature"`
|
Temperature float64 `yaml:"temperature" json:"temperature"`
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
id: aion-2
|
id: aion-2
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: aion-labs/aion-2.0
|
model: aion-labs/aion-2.0
|
||||||
temperature: 0.72
|
temperature: 0.72
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
top_p: 0.95
|
top_p: 0.95
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: claude-fable-latest
|
id: claude-fable-latest
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "~anthropic/claude-fable-latest"
|
model: "~anthropic/claude-fable-latest"
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
timeout_seconds: 600
|
timeout_seconds: 600
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: claude-haiku-latest
|
id: claude-haiku-latest
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "~anthropic/claude-haiku-latest"
|
model: "~anthropic/claude-haiku-latest"
|
||||||
reasoning_effort: medium
|
reasoning_effort: medium
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: claude-opus-latest
|
id: claude-opus-latest
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "~anthropic/claude-opus-latest"
|
model: "~anthropic/claude-opus-latest"
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: claude-sonnet-latest
|
id: claude-sonnet-latest
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "~anthropic/claude-sonnet-latest"
|
model: "~anthropic/claude-sonnet-latest"
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: deepseek-3-2
|
id: deepseek-3-2
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: deepseek/deepseek-v3.2
|
model: deepseek/deepseek-v3.2
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: deepseek-4-flash
|
id: deepseek-4-flash
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: deepseek/deepseek-v4-flash
|
model: deepseek/deepseek-v4-flash
|
||||||
#reasoning_effort: medium
|
#reasoning_effort: medium
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: deepseek-4-pro
|
id: deepseek-4-pro
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: deepseek/deepseek-v4-pro
|
model: deepseek/deepseek-v4-pro
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
id: gemini-2-flash-lite
|
id: gemini-2-flash-lite
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "google/gemini-2.5-flash-lite"
|
model: "google/gemini-2.5-flash-lite"
|
||||||
#temperature: 0.15
|
#temperature: 0.15
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
#top_p: 0.98
|
#top_p: 0.98
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
id: gemini-2-flash
|
id: gemini-2-flash
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "google/gemini-2.5-flash"
|
model: "google/gemini-2.5-flash"
|
||||||
#temperature: 0.15
|
#temperature: 0.15
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
#top_p: 0.98
|
#top_p: 0.98
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
id: gemini-2-pro
|
id: gemini-2-pro
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "google/gemini-2.5-pro"
|
model: "google/gemini-2.5-pro"
|
||||||
#temperature: 0.15
|
#temperature: 0.15
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
#top_p: 0.98
|
#top_p: 0.98
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
id: gemini-3-flash-lite
|
id: gemini-3-flash-lite
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "google/gemini-3.1-flash-lite"
|
model: "google/gemini-3.1-flash-lite"
|
||||||
#temperature: 0.15
|
#temperature: 0.15
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
#top_p: 0.98
|
#top_p: 0.98
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
id: gemini-flash-latest
|
id: gemini-flash-latest
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "~google/gemini-flash-latest"
|
model: "~google/gemini-flash-latest"
|
||||||
#temperature: 0.15
|
#temperature: 0.15
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
#top_p: 0.98
|
#top_p: 0.98
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
id: gemini-pro-latest
|
id: gemini-pro-latest
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "~google/gemini-pro-latest"
|
model: "~google/gemini-pro-latest"
|
||||||
#temperature: 0.15
|
#temperature: 0.15
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
#top_p: 0.98
|
#top_p: 0.98
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
id: gemma-4-31b
|
id: gemma-4-31b
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: google/gemma-4-31b-it:exacto
|
model: google/gemma-4-31b-it:exacto
|
||||||
temperature: 0.15
|
temperature: 0.15
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
top_p: 0.98
|
top_p: 0.98
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
id: minimax-m2
|
id: minimax-m2
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: minimax/minimax-m2.5
|
model: minimax/minimax-m2.5
|
||||||
temperature: 0.5
|
temperature: 0.5
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
top_p: 0.95
|
top_p: 0.95
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,9 +1,8 @@
|
|||||||
id: minimax-m3
|
id: minimax-m3
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: minimax/minimax-m3
|
model: minimax/minimax-m3
|
||||||
#temperature: 0.5
|
#temperature: 0.5
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
#top_p: 0.95
|
#top_p: 0.95
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: mistral-large-2512
|
id: mistral-large-2512
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: mistralai/mistral-large-2512
|
model: mistralai/mistral-large-2512
|
||||||
temperature: 0.15
|
temperature: 0.15
|
||||||
top_p: 0.98
|
top_p: 0.98
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
|
|||||||
@@ -1,8 +1,7 @@
|
|||||||
id: mistral-medium-3-5
|
id: mistral-medium-3-5
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: mistralai/mistral-medium-3-5
|
model: mistralai/mistral-medium-3-5
|
||||||
temperature: 0.15
|
temperature: 0.15
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
top_p: 0.98
|
top_p: 0.98
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: mistral-small-3
|
id: mistral-small-3
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: mistralai/mistral-small-3.2-24b-instruct
|
model: mistralai/mistral-small-3.2-24b-instruct
|
||||||
temperature: 0.05
|
temperature: 0.05
|
||||||
top_p: 1.0
|
top_p: 1.0
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
|
|||||||
@@ -1,8 +1,7 @@
|
|||||||
id: mistral-small-4
|
id: mistral-small-4
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: mistralai/mistral-small-2603
|
model: mistralai/mistral-small-2603
|
||||||
temperature: 0.1
|
temperature: 0.1
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
top_p: 0.98
|
top_p: 0.98
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: nemotron-3-ultra
|
id: nemotron-3-ultra
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: nvidia/nemotron-3-ultra-550b-a55b
|
model: nvidia/nemotron-3-ultra-550b-a55b
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
timeout_seconds: 180
|
timeout_seconds: 180
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: gpt-5-mini
|
id: gpt-5-mini
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "openai/gpt-5.4-mini"
|
model: "openai/gpt-5.4-mini"
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -1,7 +1,6 @@
|
|||||||
id: gpt-5-nano
|
id: gpt-5-nano
|
||||||
endpoint: https://openrouter.ai/api/v1
|
backend: openrouter
|
||||||
model: "openai/gpt-5.4-nano"
|
model: "openai/gpt-5.4-nano"
|
||||||
reasoning_effort: high
|
reasoning_effort: high
|
||||||
timeout_seconds: 240
|
timeout_seconds: 240
|
||||||
api_key_env: OPENROUTER_API_KEY
|
|
||||||
service_tier: flex
|
service_tier: flex
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ import (
|
|||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v3"
|
||||||
@@ -28,6 +29,12 @@ func TestBuiltInProfilesValidateThroughRepository(t *testing.T) {
|
|||||||
if p.ID != id {
|
if p.ID != id {
|
||||||
t.Fatalf("expected profile id %q, got %q", id, p.ID)
|
t.Fatalf("expected profile id %q, got %q", id, p.ID)
|
||||||
}
|
}
|
||||||
|
if p.BackendID != backend.OpenRouterID {
|
||||||
|
t.Fatalf("expected profile %q to select %q, got %q", id, backend.OpenRouterID, p.BackendID)
|
||||||
|
}
|
||||||
|
if p.Endpoint != "" || p.APIKeyEnv != "" {
|
||||||
|
t.Fatalf("expected profile %q to inherit backend connection settings, got endpoint=%q api_key_env=%q", id, p.Endpoint, p.APIKeyEnv)
|
||||||
|
}
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -60,6 +67,15 @@ func loadBuiltInProfileIDs(t *testing.T) map[string]string {
|
|||||||
if _, ok := raw["api_key"]; ok {
|
if _, ok := raw["api_key"]; ok {
|
||||||
t.Fatalf("built-in profile %s contains raw api_key", name)
|
t.Fatalf("built-in profile %s contains raw api_key", name)
|
||||||
}
|
}
|
||||||
|
if raw["backend"] != backend.OpenRouterID {
|
||||||
|
t.Fatalf("built-in profile %s does not select %q", name, backend.OpenRouterID)
|
||||||
|
}
|
||||||
|
if _, ok := raw["endpoint"]; ok {
|
||||||
|
t.Fatalf("built-in profile %s repeats endpoint", name)
|
||||||
|
}
|
||||||
|
if _, ok := raw["api_key_env"]; ok {
|
||||||
|
t.Fatalf("built-in profile %s repeats api_key_env", name)
|
||||||
|
}
|
||||||
id, ok := raw["id"].(string)
|
id, ok := raw["id"].(string)
|
||||||
if !ok || strings.TrimSpace(id) == "" {
|
if !ok || strings.TrimSpace(id) == "" {
|
||||||
t.Fatalf("built-in profile %s has missing id", name)
|
t.Fatalf("built-in profile %s has missing id", name)
|
||||||
|
|||||||
@@ -121,6 +121,7 @@ func loadProfile(ctx context.Context, fsys fs.FS, root string, id string) (*doma
|
|||||||
if prof.ID != id {
|
if prof.ID != id {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
prof.BackendID = strings.TrimSpace(prof.BackendID)
|
||||||
if err := validateProfile(&prof); err != nil {
|
if err := validateProfile(&prof); err != nil {
|
||||||
if errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
if errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
||||||
return nil, fmt.Errorf("%w: %s", err, relPath)
|
return nil, fmt.Errorf("%w: %s", err, relPath)
|
||||||
@@ -189,8 +190,8 @@ func validateProfile(p *domain.ExecutionProfile) error {
|
|||||||
if strings.TrimSpace(p.ID) == "" {
|
if strings.TrimSpace(p.ID) == "" {
|
||||||
return errors.New("id is required")
|
return errors.New("id is required")
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(p.Endpoint) == "" {
|
if strings.TrimSpace(p.BackendID) == "" && strings.TrimSpace(p.Endpoint) == "" {
|
||||||
return errors.New("endpoint is required")
|
return errors.New("backend or endpoint is required")
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(p.Model) == "" {
|
if strings.TrimSpace(p.Model) == "" {
|
||||||
return errors.New("model is required")
|
return errors.New("model is required")
|
||||||
|
|||||||
@@ -52,6 +52,43 @@ func TestFilesystemRepository_GetProfile(t *testing.T) {
|
|||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
|
t.Run("backend and endpoint connection matrix", func(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
connection string
|
||||||
|
wantBackend string
|
||||||
|
wantEndpoint string
|
||||||
|
wantErr bool
|
||||||
|
}{
|
||||||
|
{name: "backend only", connection: "backend: ' openrouter '", wantBackend: "openrouter"},
|
||||||
|
{name: "endpoint only", connection: "endpoint: http://localhost:8000/v1", wantEndpoint: "http://localhost:8000/v1"},
|
||||||
|
{name: "both", connection: "backend: openrouter\nendpoint: http://localhost:8000/v1", wantBackend: "openrouter", wantEndpoint: "http://localhost:8000/v1"},
|
||||||
|
{name: "neither", wantErr: true},
|
||||||
|
{name: "blank backend", connection: "backend: ' '", wantErr: true},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, tt := range tests {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
id := "connection-" + strings.ReplaceAll(tt.name, " ", "-")
|
||||||
|
writeProfileTestFile(t, filepath.Join(tmpDir, id+".yaml"), "id: "+id+"\nmodel: model\n"+tt.connection+"\n")
|
||||||
|
|
||||||
|
p, err := repo.GetProfile(ctx, id)
|
||||||
|
if tt.wantErr {
|
||||||
|
if !errors.Is(err, ErrInvalidProfile) {
|
||||||
|
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
||||||
|
}
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("expected profile to load, got %v", err)
|
||||||
|
}
|
||||||
|
if p.BackendID != tt.wantBackend || p.Endpoint != tt.wantEndpoint {
|
||||||
|
t.Fatalf("unexpected connection values: backend=%q endpoint=%q", p.BackendID, p.Endpoint)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
t.Run("valid profile with api_key_env", func(t *testing.T) {
|
t.Run("valid profile with api_key_env", func(t *testing.T) {
|
||||||
p, err := repo.GetProfile(ctx, "local-secure")
|
p, err := repo.GetProfile(ctx, "local-secure")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
@@ -40,6 +40,7 @@ var (
|
|||||||
type Runner struct {
|
type Runner struct {
|
||||||
promptDefs promptdef.Repository
|
promptDefs promptdef.Repository
|
||||||
profiles profile.Repository
|
profiles profile.Repository
|
||||||
|
backends BackendResolver
|
||||||
artifacts artifact.Reader
|
artifacts artifact.Reader
|
||||||
renderer prompt.Renderer
|
renderer prompt.Renderer
|
||||||
llm llm.Client
|
llm llm.Client
|
||||||
@@ -47,20 +48,27 @@ type Runner struct {
|
|||||||
repairer OutputRepairer
|
repairer OutputRepairer
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// BackendResolver resolves one normalized backend ID.
|
||||||
|
type BackendResolver interface {
|
||||||
|
GetBackend(string) (domain.Backend, error)
|
||||||
|
}
|
||||||
|
|
||||||
func NewRunner(
|
func NewRunner(
|
||||||
promptDefs promptdef.Repository,
|
promptDefs promptdef.Repository,
|
||||||
profiles profile.Repository,
|
profiles profile.Repository,
|
||||||
|
backends BackendResolver,
|
||||||
artifacts artifact.Reader,
|
artifacts artifact.Reader,
|
||||||
renderer prompt.Renderer,
|
renderer prompt.Renderer,
|
||||||
llmClient llm.Client,
|
llmClient llm.Client,
|
||||||
validator validate.Validator,
|
validator validate.Validator,
|
||||||
) *Runner {
|
) *Runner {
|
||||||
return NewRunnerWithRepairer(promptDefs, profiles, artifacts, renderer, llmClient, validator, nil)
|
return NewRunnerWithRepairer(promptDefs, profiles, backends, artifacts, renderer, llmClient, validator, nil)
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewRunnerWithRepairer(
|
func NewRunnerWithRepairer(
|
||||||
promptDefs promptdef.Repository,
|
promptDefs promptdef.Repository,
|
||||||
profiles profile.Repository,
|
profiles profile.Repository,
|
||||||
|
backends BackendResolver,
|
||||||
artifacts artifact.Reader,
|
artifacts artifact.Reader,
|
||||||
renderer prompt.Renderer,
|
renderer prompt.Renderer,
|
||||||
llmClient llm.Client,
|
llmClient llm.Client,
|
||||||
@@ -70,6 +78,7 @@ func NewRunnerWithRepairer(
|
|||||||
return &Runner{
|
return &Runner{
|
||||||
promptDefs: promptDefs,
|
promptDefs: promptDefs,
|
||||||
profiles: profiles,
|
profiles: profiles,
|
||||||
|
backends: backends,
|
||||||
artifacts: artifacts,
|
artifacts: artifacts,
|
||||||
renderer: renderer,
|
renderer: renderer,
|
||||||
llm: llmClient,
|
llm: llmClient,
|
||||||
@@ -153,6 +162,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
|||||||
PromptHash: prepared.PromptHash,
|
PromptHash: prepared.PromptHash,
|
||||||
RenderedPromptHash: prepared.RenderedPromptHash,
|
RenderedPromptHash: prepared.RenderedPromptHash,
|
||||||
SelectedProfileID: prepared.SelectedProfileID,
|
SelectedProfileID: prepared.SelectedProfileID,
|
||||||
|
SelectedBackendID: prepared.SelectedBackendID,
|
||||||
ModelName: prepared.EffectiveModelParams.Model,
|
ModelName: prepared.EffectiveModelParams.Model,
|
||||||
Endpoint: prepared.EffectiveModelParams.Endpoint,
|
Endpoint: prepared.EffectiveModelParams.Endpoint,
|
||||||
EffectiveModelParams: prepared.EffectiveModelParams,
|
EffectiveModelParams: prepared.EffectiveModelParams,
|
||||||
@@ -193,7 +203,20 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
|
|||||||
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
effectiveModel, targetPresence, err := resolveExecutionTarget(execProfile, req.Execution)
|
var selectedBackend *domain.Backend
|
||||||
|
if backendID := strings.TrimSpace(execProfile.BackendID); backendID != "" {
|
||||||
|
execProfile.BackendID = backendID
|
||||||
|
if r.backends == nil {
|
||||||
|
return nil, fmt.Errorf("%w: backend %q cannot be resolved", ErrProfileLoad, backendID)
|
||||||
|
}
|
||||||
|
resolvedBackend, resolveErr := r.backends.GetBackend(backendID)
|
||||||
|
if resolveErr != nil {
|
||||||
|
return nil, fmt.Errorf("%w: backend %q: %w", ErrProfileLoad, backendID, resolveErr)
|
||||||
|
}
|
||||||
|
selectedBackend = &resolvedBackend
|
||||||
|
}
|
||||||
|
|
||||||
|
effectiveModel, targetPresence, err := resolveExecutionTarget(selectedBackend, execProfile, req.Execution)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
||||||
}
|
}
|
||||||
@@ -239,6 +262,7 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
|
|||||||
PromptVersion: def.Version,
|
PromptVersion: def.Version,
|
||||||
PromptHash: promptDefinitionHash,
|
PromptHash: promptDefinitionHash,
|
||||||
SelectedProfileID: selectedProfileID,
|
SelectedProfileID: selectedProfileID,
|
||||||
|
SelectedBackendID: effectiveModel.BackendID,
|
||||||
EffectiveModelParams: effectiveModel,
|
EffectiveModelParams: effectiveModel,
|
||||||
TargetPresence: targetPresence,
|
TargetPresence: targetPresence,
|
||||||
OutputContract: effectiveContract,
|
OutputContract: effectiveContract,
|
||||||
@@ -338,6 +362,9 @@ func (r *Runner) shouldAttemptRepair(contract domain.OutputContract, validationR
|
|||||||
|
|
||||||
func mergeExecutionTarget(base domain.ExecutionTarget, override domain.ExecutionTarget) domain.ExecutionTarget {
|
func mergeExecutionTarget(base domain.ExecutionTarget, override domain.ExecutionTarget) domain.ExecutionTarget {
|
||||||
out := base
|
out := base
|
||||||
|
if strings.TrimSpace(override.BackendID) != "" {
|
||||||
|
out.BackendID = override.BackendID
|
||||||
|
}
|
||||||
if override.Endpoint != "" {
|
if override.Endpoint != "" {
|
||||||
out.Endpoint = override.Endpoint
|
out.Endpoint = override.Endpoint
|
||||||
}
|
}
|
||||||
@@ -367,6 +394,7 @@ func mergeExecutionTarget(base domain.ExecutionTarget, override domain.Execution
|
|||||||
}
|
}
|
||||||
if override.APIKeyRequired {
|
if override.APIKeyRequired {
|
||||||
out.APIKeyRequired = true
|
out.APIKeyRequired = true
|
||||||
|
out.APIKeyEnv = ""
|
||||||
}
|
}
|
||||||
if len(override.ExtraParams) > 0 {
|
if len(override.ExtraParams) > 0 {
|
||||||
out.ExtraParams = copyExtraParams(override.ExtraParams)
|
out.ExtraParams = copyExtraParams(override.ExtraParams)
|
||||||
@@ -426,8 +454,9 @@ func mergeExecutionTargetOverride(base domain.ExecutionTarget, override domain.E
|
|||||||
return out, presence, nil
|
return out, presence, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) {
|
func resolveExecutionTarget(backendValue *domain.Backend, profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) {
|
||||||
out := defaults.ExecutionTargetDefault()
|
out := defaults.ExecutionTargetDefault()
|
||||||
|
out = mergeExecutionTarget(out, backendToTarget(backendValue))
|
||||||
out = mergeExecutionTarget(out, executionProfileToTarget(profileValue))
|
out = mergeExecutionTarget(out, executionProfileToTarget(profileValue))
|
||||||
var presence domain.ExecutionTargetPresence
|
var presence domain.ExecutionTargetPresence
|
||||||
if override != nil {
|
if override != nil {
|
||||||
@@ -461,8 +490,13 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
|
|||||||
if p == nil {
|
if p == nil {
|
||||||
return domain.ExecutionTarget{}
|
return domain.ExecutionTarget{}
|
||||||
}
|
}
|
||||||
|
endpoint := p.Endpoint
|
||||||
|
if strings.TrimSpace(endpoint) == "" {
|
||||||
|
endpoint = ""
|
||||||
|
}
|
||||||
return domain.ExecutionTarget{
|
return domain.ExecutionTarget{
|
||||||
Endpoint: p.Endpoint,
|
BackendID: p.BackendID,
|
||||||
|
Endpoint: endpoint,
|
||||||
Model: p.Model,
|
Model: p.Model,
|
||||||
Temperature: p.Temperature,
|
Temperature: p.Temperature,
|
||||||
MaxTokens: p.MaxTokens,
|
MaxTokens: p.MaxTokens,
|
||||||
@@ -476,6 +510,18 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func backendToTarget(value *domain.Backend) domain.ExecutionTarget {
|
||||||
|
if value == nil {
|
||||||
|
return domain.ExecutionTarget{}
|
||||||
|
}
|
||||||
|
return domain.ExecutionTarget{
|
||||||
|
BackendID: value.ID,
|
||||||
|
Endpoint: value.Endpoint,
|
||||||
|
APIKeyEnv: value.APIKeyEnv,
|
||||||
|
ExtraParams: copyExtraParams(value.ExtraParams),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func copyExtraParams(src map[string]any) map[string]any {
|
func copyExtraParams(src map[string]any) map[string]any {
|
||||||
if len(src) == 0 {
|
if len(src) == 0 {
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -34,6 +34,18 @@ type fakeExecutionProfileRepo struct {
|
|||||||
lastID string
|
lastID string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type fakeBackendResolver struct {
|
||||||
|
backends map[string]domain.Backend
|
||||||
|
}
|
||||||
|
|
||||||
|
func (f fakeBackendResolver) GetBackend(id string) (domain.Backend, error) {
|
||||||
|
value, ok := f.backends[id]
|
||||||
|
if !ok {
|
||||||
|
return domain.Backend{}, errors.New("backend not found")
|
||||||
|
}
|
||||||
|
return value, nil
|
||||||
|
}
|
||||||
|
|
||||||
func (f *fakeExecutionProfileRepo) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
func (f *fakeExecutionProfileRepo) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||||
f.lastID = id
|
f.lastID = id
|
||||||
if f.err != nil {
|
if f.err != nil {
|
||||||
@@ -167,7 +179,7 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) {
|
|||||||
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
||||||
llmClient := &fakeLLM{forbid: true}
|
llmClient := &fakeLLM{forbid: true}
|
||||||
|
|
||||||
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil)
|
||||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
PromptVersion: "1",
|
PromptVersion: "1",
|
||||||
@@ -257,7 +269,7 @@ func TestRunnerPrepareSelectedProfileDoesNotExistFails(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerPreparePromptLoadFailure(t *testing.T) {
|
func TestRunnerPreparePromptLoadFailure(t *testing.T) {
|
||||||
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
|
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
|
||||||
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p"})
|
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p"})
|
||||||
if !errors.Is(err, ErrPromptLoad) {
|
if !errors.Is(err, ErrPromptLoad) {
|
||||||
t.Fatalf("expected ErrPromptLoad, got %v", err)
|
t.Fatalf("expected ErrPromptLoad, got %v", err)
|
||||||
@@ -281,7 +293,7 @@ func TestRunnerPrepareRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) {
|
|||||||
ServiceTier: "priority",
|
ServiceTier: "priority",
|
||||||
},
|
},
|
||||||
}}
|
}}
|
||||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||||
|
|
||||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -379,12 +391,12 @@ func TestRunnerPrepareRequestNumericOverridePresence(t *testing.T) {
|
|||||||
TopP: 0.8,
|
TopP: 0.8,
|
||||||
TimeoutSeconds: 45,
|
TimeoutSeconds: 45,
|
||||||
},
|
},
|
||||||
}},
|
}}, nil,
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
defaultRenderer(),
|
defaultRenderer(),
|
||||||
&fakeLLM{forbid: true},
|
&fakeLLM{forbid: true},
|
||||||
nil,
|
nil)
|
||||||
)
|
|
||||||
|
|
||||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -426,12 +438,12 @@ func TestRunnerPrepareInvalidRequestNumericOverridesFail(t *testing.T) {
|
|||||||
t.Run(tc.name, func(t *testing.T) {
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
defaultRenderer(),
|
defaultRenderer(),
|
||||||
&fakeLLM{forbid: true},
|
&fakeLLM{forbid: true},
|
||||||
nil,
|
nil)
|
||||||
)
|
|
||||||
|
|
||||||
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -458,7 +470,7 @@ func TestRunnerPrepareSelectedProfileBeatsBuiltInDefault(t *testing.T) {
|
|||||||
ServiceTier: "priority",
|
ServiceTier: "priority",
|
||||||
},
|
},
|
||||||
}}
|
}}
|
||||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||||
|
|
||||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -495,12 +507,12 @@ func TestRunnerPrepareFileBackedPromptBodiesRenderCorrectly(t *testing.T) {
|
|||||||
llmClient := &fakeLLM{forbid: true}
|
llmClient := &fakeLLM{forbid: true}
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
promptdef.NewFilesystemRepository(promptDir),
|
promptdef.NewFilesystemRepository(promptDir),
|
||||||
profile.NewFilesystemRepository(profileDir),
|
profile.NewFilesystemRepository(profileDir), nil,
|
||||||
|
|
||||||
reader,
|
reader,
|
||||||
prompt.NewGoRenderer(),
|
prompt.NewGoRenderer(),
|
||||||
llmClient,
|
llmClient,
|
||||||
nil,
|
nil)
|
||||||
)
|
|
||||||
|
|
||||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
PromptID: "valid-file-backed",
|
PromptID: "valid-file-backed",
|
||||||
@@ -526,12 +538,13 @@ func TestRunnerPrepareRequiredInputMissingFails(t *testing.T) {
|
|||||||
|
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: def},
|
&fakePromptRepo{def: def},
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
prompt.NewGoRenderer(),
|
prompt.NewGoRenderer(),
|
||||||
&fakeLLM{forbid: true},
|
&fakeLLM{forbid: true},
|
||||||
nil,
|
nil)
|
||||||
)
|
|
||||||
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
@@ -552,12 +565,13 @@ func TestRunnerPrepareUnknownTemplateInputReferenceFails(t *testing.T) {
|
|||||||
|
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: def},
|
&fakePromptRepo{def: def},
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
prompt.NewGoRenderer(),
|
prompt.NewGoRenderer(),
|
||||||
&fakeLLM{forbid: true},
|
&fakeLLM{forbid: true},
|
||||||
nil,
|
nil)
|
||||||
)
|
|
||||||
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
@@ -579,7 +593,7 @@ func TestRunnerPrepareAPIKeyEnvNameIncludedButNotResolvedValue(t *testing.T) {
|
|||||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName},
|
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName},
|
||||||
}}
|
}}
|
||||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||||
|
|
||||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -607,12 +621,12 @@ func TestRunnerPrepareJSONSchemaBuildsStructuredOutputSpec(t *testing.T) {
|
|||||||
}
|
}
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: def},
|
&fakePromptRepo{def: def},
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
defaultRenderer(),
|
defaultRenderer(),
|
||||||
&fakeLLM{forbid: true},
|
&fakeLLM{forbid: true},
|
||||||
validator,
|
validator)
|
||||||
)
|
|
||||||
|
|
||||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -651,12 +665,12 @@ func TestRunnerPrepareJSONSchemaSchemaLoadFailureReturnsValidationError(t *testi
|
|||||||
validator := &fakeValidator{schemaErr: errors.New("schema unavailable")}
|
validator := &fakeValidator{schemaErr: errors.New("schema unavailable")}
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: def},
|
&fakePromptRepo{def: def},
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
defaultRenderer(),
|
defaultRenderer(),
|
||||||
&fakeLLM{forbid: true},
|
&fakeLLM{forbid: true},
|
||||||
validator,
|
validator)
|
||||||
)
|
|
||||||
|
|
||||||
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -681,12 +695,12 @@ func TestRunnerRunJSONSchemaSchemaLoadFailureFailsBeforeLLM(t *testing.T) {
|
|||||||
validator := &fakeValidator{schemaErr: errors.New("schema unavailable")}
|
validator := &fakeValidator{schemaErr: errors.New("schema unavailable")}
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: def},
|
&fakePromptRepo{def: def},
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
defaultRenderer(),
|
defaultRenderer(),
|
||||||
llmClient,
|
llmClient,
|
||||||
validator,
|
validator)
|
||||||
)
|
|
||||||
|
|
||||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -836,7 +850,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
|
|||||||
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
||||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}}
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}}
|
||||||
|
|
||||||
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil)
|
||||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
PromptVersion: "1",
|
PromptVersion: "1",
|
||||||
@@ -915,7 +929,7 @@ func TestRunnerRunPassesExtraParamsToGenerateRequestTarget(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}}
|
}}
|
||||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||||
|
|
||||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -941,7 +955,7 @@ func TestRunnerRunAndPrepareResolveSameProfileAndEffectiveSettings(t *testing.T)
|
|||||||
}}
|
}}
|
||||||
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
||||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap"}}
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap"}}
|
||||||
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil)
|
||||||
|
|
||||||
req := domain.RunRequest{
|
req := domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -1071,7 +1085,7 @@ func TestRunnerRunExplicitRuntimeOverrideBeatsSelectedProfileValue(t *testing.T)
|
|||||||
},
|
},
|
||||||
}}
|
}}
|
||||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||||
|
|
||||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -1115,7 +1129,7 @@ func TestRunnerRunSelectedProfileBeatsBuiltInDefault(t *testing.T) {
|
|||||||
},
|
},
|
||||||
}}
|
}}
|
||||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||||
|
|
||||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -1142,7 +1156,7 @@ func TestRunnerRunBuiltInDefaultsUsedWhenProfileOmitsOptionalFields(t *testing.T
|
|||||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"},
|
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"},
|
||||||
}}
|
}}
|
||||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||||
|
|
||||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -1172,7 +1186,7 @@ func TestRunnerRunAPIKeyEnvResolvesFromEnvironment(t *testing.T) {
|
|||||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_TEST_API_KEY"},
|
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_TEST_API_KEY"},
|
||||||
}}
|
}}
|
||||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||||
|
|
||||||
res, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
res, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1188,7 +1202,7 @@ func TestRunnerRunAPIKeyEnvMissingEnvironmentValueFailsClearly(t *testing.T) {
|
|||||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_MISSING_KEY"},
|
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_MISSING_KEY"},
|
||||||
}}
|
}}
|
||||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||||
|
|
||||||
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||||
if !errors.Is(err, ErrInvalidRequest) {
|
if !errors.Is(err, ErrInvalidRequest) {
|
||||||
@@ -1209,7 +1223,7 @@ func TestRunnerRunDirectAPIKeyBypassesMissingEnvAndReachesLLM(t *testing.T) {
|
|||||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_MISSING_KEY"},
|
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_MISSING_KEY"},
|
||||||
}}
|
}}
|
||||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||||
|
|
||||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -1233,7 +1247,7 @@ func TestRunnerPrepareAPIKeyRequiredFailsWithoutDirectKey(t *testing.T) {
|
|||||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true},
|
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true},
|
||||||
}}
|
}}
|
||||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||||
|
|
||||||
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -1252,7 +1266,7 @@ func TestRunnerRunAPIKeyRequiredSucceedsWithDirectKey(t *testing.T) {
|
|||||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true},
|
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true},
|
||||||
}}
|
}}
|
||||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||||
|
|
||||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -1279,7 +1293,7 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideWorks(t *testing.T) {
|
|||||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"},
|
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"},
|
||||||
}}
|
}}
|
||||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||||
|
|
||||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -1304,7 +1318,7 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideBeatsProfile(t *testing.T) {
|
|||||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: profileEnv},
|
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: profileEnv},
|
||||||
}}
|
}}
|
||||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||||
|
|
||||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -1328,7 +1342,7 @@ func TestRunnerRunAPIKeyValueNotPresentInMetadata(t *testing.T) {
|
|||||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName},
|
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName},
|
||||||
}}
|
}}
|
||||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||||
|
|
||||||
res, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
res, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -1344,7 +1358,7 @@ func TestRunnerRunAPIKeyValueNotPresentInMetadata(t *testing.T) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerRunPromptLoadFailure(t *testing.T) {
|
func TestRunnerRunPromptLoadFailure(t *testing.T) {
|
||||||
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
|
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
|
||||||
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"})
|
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"})
|
||||||
if !errors.Is(err, ErrPromptLoad) {
|
if !errors.Is(err, ErrPromptLoad) {
|
||||||
t.Fatalf("expected ErrPromptLoad, got %v", err)
|
t.Fatalf("expected ErrPromptLoad, got %v", err)
|
||||||
@@ -1357,12 +1371,12 @@ func TestRunnerRunPromptLoadFailure(t *testing.T) {
|
|||||||
func TestRunnerRunArtifactLoadFailure(t *testing.T) {
|
func TestRunnerRunArtifactLoadFailure(t *testing.T) {
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||||
|
|
||||||
&fakeArtifactReader{errByURI: map[string]error{"a://bad": errors.New("read failed")}},
|
&fakeArtifactReader{errByURI: map[string]error{"a://bad": errors.New("read failed")}},
|
||||||
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
|
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
|
||||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
|
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
|
||||||
nil,
|
nil)
|
||||||
)
|
|
||||||
|
|
||||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -1377,12 +1391,13 @@ func TestRunnerRunArtifactLoadFailure(t *testing.T) {
|
|||||||
func TestRunnerRunPromptRenderFailure(t *testing.T) {
|
func TestRunnerRunPromptRenderFailure(t *testing.T) {
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
&fakeRenderer{err: errors.New("render failed")},
|
&fakeRenderer{err: errors.New("render failed")},
|
||||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
|
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
|
||||||
nil,
|
nil)
|
||||||
)
|
|
||||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
@@ -1396,12 +1411,13 @@ func TestRunnerRunPromptRenderFailure(t *testing.T) {
|
|||||||
func TestRunnerRunLLMFailure(t *testing.T) {
|
func TestRunnerRunLLMFailure(t *testing.T) {
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
defaultRenderer(),
|
defaultRenderer(),
|
||||||
&fakeLLM{err: errors.New("llm failed")},
|
&fakeLLM{err: errors.New("llm failed")},
|
||||||
nil,
|
nil)
|
||||||
)
|
|
||||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
@@ -1415,12 +1431,13 @@ func TestRunnerRunLLMFailure(t *testing.T) {
|
|||||||
func TestRunnerRunCancellationPreservesGenerationCategory(t *testing.T) {
|
func TestRunnerRunCancellationPreservesGenerationCategory(t *testing.T) {
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
defaultRenderer(),
|
defaultRenderer(),
|
||||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "ignored"}},
|
&fakeLLM{resp: &domain.GenerateResponse{Content: "ignored"}},
|
||||||
nil,
|
nil)
|
||||||
)
|
|
||||||
ctx, cancel := context.WithCancel(context.Background())
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
cancel()
|
cancel()
|
||||||
|
|
||||||
@@ -1440,12 +1457,13 @@ func TestRunnerRunCancellationPreservesGenerationCategory(t *testing.T) {
|
|||||||
func TestRunnerRunLLMInvalidRequestMapsToUsecaseInvalidRequest(t *testing.T) {
|
func TestRunnerRunLLMInvalidRequestMapsToUsecaseInvalidRequest(t *testing.T) {
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
defaultRenderer(),
|
defaultRenderer(),
|
||||||
&fakeLLM{err: llm.ErrInvalidRequest},
|
&fakeLLM{err: llm.ErrInvalidRequest},
|
||||||
nil,
|
nil)
|
||||||
)
|
|
||||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
@@ -1463,12 +1481,13 @@ func TestRunnerRunValidationStillWorks(t *testing.T) {
|
|||||||
validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}}
|
validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}}
|
||||||
runner := NewRunner(
|
runner := NewRunner(
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
|
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
defaultRenderer(),
|
defaultRenderer(),
|
||||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}},
|
&fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}},
|
||||||
validator,
|
validator)
|
||||||
)
|
|
||||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
@@ -1489,14 +1508,17 @@ func TestRunnerRunStructuredRepairRemainsBoundedAndUsesEffectiveModelSettings(t
|
|||||||
runner := NewRunnerWithRepairer(
|
runner := NewRunnerWithRepairer(
|
||||||
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
|
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", TimeoutSeconds: 55},
|
"exec": {ID: "exec", BackendID: "custom", Model: "profile-model", TimeoutSeconds: 55},
|
||||||
|
}}, fakeBackendResolver{backends: map[string]domain.Backend{
|
||||||
|
"custom": {ID: "custom", Endpoint: "http://backend/v1"},
|
||||||
}},
|
}},
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
defaultRenderer(),
|
defaultRenderer(),
|
||||||
llmClient,
|
llmClient,
|
||||||
validate.NewStandardValidator("."),
|
validate.NewStandardValidator("."),
|
||||||
repairer,
|
repairer)
|
||||||
)
|
|
||||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
ProfileID: "exec",
|
ProfileID: "exec",
|
||||||
@@ -1518,6 +1540,12 @@ func TestRunnerRunStructuredRepairRemainsBoundedAndUsesEffectiveModelSettings(t
|
|||||||
if repairer.reqs[0].Target.TimeoutSeconds != 22 {
|
if repairer.reqs[0].Target.TimeoutSeconds != 22 {
|
||||||
t.Fatalf("expected repair to use effective timeout, got %d", repairer.reqs[0].Target.TimeoutSeconds)
|
t.Fatalf("expected repair to use effective timeout, got %d", repairer.reqs[0].Target.TimeoutSeconds)
|
||||||
}
|
}
|
||||||
|
if llmClient.lastReq.Target.BackendID != "custom" ||
|
||||||
|
repairer.reqs[0].Target.BackendID != "custom" ||
|
||||||
|
res.SelectedBackendID != "custom" {
|
||||||
|
t.Fatalf("expected backend identity in generation, repair, and result: generate=%q repair=%q result=%q",
|
||||||
|
llmClient.lastReq.Target.BackendID, repairer.reqs[0].Target.BackendID, res.SelectedBackendID)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunnerRunJSONSchemaRepairCarriesStructuredOutputSpec(t *testing.T) {
|
func TestRunnerRunJSONSchemaRepairCarriesStructuredOutputSpec(t *testing.T) {
|
||||||
@@ -1546,13 +1574,13 @@ func TestRunnerRunJSONSchemaRepairCarriesStructuredOutputSpec(t *testing.T) {
|
|||||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: `{"events":[1]}`}}
|
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: `{"events":[1]}`}}
|
||||||
runner := NewRunnerWithRepairer(
|
runner := NewRunnerWithRepairer(
|
||||||
&fakePromptRepo{def: def},
|
&fakePromptRepo{def: def},
|
||||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
defaultRenderer(),
|
defaultRenderer(),
|
||||||
llmClient,
|
llmClient,
|
||||||
validator,
|
validator,
|
||||||
repairer,
|
repairer)
|
||||||
)
|
|
||||||
|
|
||||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||||
PromptID: "p",
|
PromptID: "p",
|
||||||
@@ -1629,13 +1657,12 @@ func TestResolveExecutionTargetProfileValuesPopulateAllSupportedFields(t *testin
|
|||||||
ServiceTier: "priority",
|
ServiceTier: "priority",
|
||||||
ReasoningEffort: "low",
|
ReasoningEffort: "low",
|
||||||
APIKeyEnv: "PROFILE_KEY",
|
APIKeyEnv: "PROFILE_KEY",
|
||||||
APIKeyRequired: true,
|
|
||||||
ExtraParams: map[string]any{
|
ExtraParams: map[string]any{
|
||||||
"profile_option": "enabled",
|
"profile_option": "enabled",
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
target, presence, err := resolveExecutionTarget(profileValue, nil)
|
target, presence, err := resolveExecutionTarget(nil, profileValue, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -1690,7 +1717,7 @@ func TestResolveExecutionTargetRuntimeOverridesBeatProfileForAllOverrideableFiel
|
|||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
target, presence, err := resolveExecutionTarget(profileValue, override)
|
target, presence, err := resolveExecutionTarget(nil, profileValue, override)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("expected no error, got %v", err)
|
t.Fatalf("expected no error, got %v", err)
|
||||||
}
|
}
|
||||||
@@ -1813,6 +1840,118 @@ func defaultExecutionProfile() *domain.ExecutionProfile {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestResolveExecutionTargetUsesBackendProfileAndRequestPrecedence(t *testing.T) {
|
||||||
|
backendValue := &domain.Backend{
|
||||||
|
ID: "custom",
|
||||||
|
Endpoint: "http://backend/v1",
|
||||||
|
APIKeyEnv: "BACKEND_KEY",
|
||||||
|
ExtraParams: map[string]any{"backend": true},
|
||||||
|
}
|
||||||
|
profileValue := &domain.ExecutionProfile{
|
||||||
|
ID: "exec",
|
||||||
|
BackendID: "custom",
|
||||||
|
Endpoint: "http://profile/v1",
|
||||||
|
Model: "profile-model",
|
||||||
|
APIKeyEnv: "PROFILE_KEY",
|
||||||
|
ExtraParams: map[string]any{"profile": true},
|
||||||
|
}
|
||||||
|
override := &domain.ExecutionTargetOverride{
|
||||||
|
Endpoint: "http://request/v1",
|
||||||
|
APIKeyEnv: "REQUEST_KEY",
|
||||||
|
ExtraParams: map[string]any{"request": true},
|
||||||
|
}
|
||||||
|
|
||||||
|
target, _, err := resolveExecutionTarget(backendValue, profileValue, override)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolve target: %v", err)
|
||||||
|
}
|
||||||
|
if target.BackendID != "custom" {
|
||||||
|
t.Fatalf("endpoint override changed backend identity: %+v", target)
|
||||||
|
}
|
||||||
|
if target.Endpoint != "http://request/v1" || target.APIKeyEnv != "REQUEST_KEY" {
|
||||||
|
t.Fatalf("request values did not win: %+v", target)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(target.ExtraParams, map[string]any{"request": true}) {
|
||||||
|
t.Fatalf("expected whole-map request replacement, got %#v", target.ExtraParams)
|
||||||
|
}
|
||||||
|
|
||||||
|
target, _, err = resolveExecutionTarget(backendValue, &domain.ExecutionProfile{
|
||||||
|
ID: "exec", BackendID: "custom", Model: "profile-model",
|
||||||
|
}, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("resolve backend defaults: %v", err)
|
||||||
|
}
|
||||||
|
if target.Endpoint != backendValue.Endpoint ||
|
||||||
|
target.APIKeyEnv != backendValue.APIKeyEnv ||
|
||||||
|
!reflect.DeepEqual(target.ExtraParams, backendValue.ExtraParams) {
|
||||||
|
t.Fatalf("backend defaults were not inherited: %+v", target)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRunnerPrepareBackendResolutionAndCredentialPrecedence(t *testing.T) {
|
||||||
|
resolver := fakeBackendResolver{backends: map[string]domain.Backend{
|
||||||
|
"custom": {ID: "custom", Endpoint: "http://backend/v1", APIKeyEnv: "BACKEND_KEY"},
|
||||||
|
}}
|
||||||
|
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
||||||
|
|
||||||
|
t.Run("unknown backend is a profile load failure", func(t *testing.T) {
|
||||||
|
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"exec": {ID: "exec", BackendID: "unknown", Model: "model"},
|
||||||
|
}}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||||
|
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||||
|
if !errors.Is(err, ErrProfileLoad) {
|
||||||
|
t.Fatalf("expected ErrProfileLoad, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("nil resolver is a profile load failure", func(t *testing.T) {
|
||||||
|
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"exec": {ID: "exec", BackendID: "custom", Model: "model"},
|
||||||
|
}}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||||
|
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||||
|
if !errors.Is(err, ErrProfileLoad) {
|
||||||
|
t.Fatalf("expected ErrProfileLoad, got %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("request environment wins", func(t *testing.T) {
|
||||||
|
t.Setenv("REQUEST_KEY", "request-secret")
|
||||||
|
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"exec": {ID: "exec", BackendID: "custom", Model: "model", APIKeyEnv: "PROFILE_KEY"},
|
||||||
|
}}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||||
|
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(),
|
||||||
|
Execution: &domain.ExecutionTargetOverride{APIKeyEnv: "REQUEST_KEY"},
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare: %v", err)
|
||||||
|
}
|
||||||
|
if prepared.EffectiveModelParams.APIKeyEnv != "REQUEST_KEY" {
|
||||||
|
t.Fatalf("unexpected credential source: %+v", prepared.EffectiveModelParams)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
t.Run("required direct key clears inherited environment", func(t *testing.T) {
|
||||||
|
t.Setenv("BACKEND_KEY", "backend-secret")
|
||||||
|
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||||
|
"exec": {ID: "exec", BackendID: "custom", Model: "model", APIKeyRequired: true},
|
||||||
|
}}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||||
|
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||||
|
if !errors.Is(err, ErrAPIKeyRequired) {
|
||||||
|
t.Fatalf("expected ErrAPIKeyRequired, got %v", err)
|
||||||
|
}
|
||||||
|
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||||
|
PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(), APIKey: "direct-secret",
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("prepare with direct key: %v", err)
|
||||||
|
}
|
||||||
|
if prepared.EffectiveModelParams.APIKeyEnv != "" {
|
||||||
|
t.Fatalf("expected inherited environment name to be cleared, got %q", prepared.EffectiveModelParams.APIKeyEnv)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
func defaultArtifactReader() *fakeArtifactReader {
|
func defaultArtifactReader() *fakeArtifactReader {
|
||||||
return &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{
|
return &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{
|
||||||
"a://ok": {Body: []byte("x"), Hash: hashString("x")},
|
"a://ok": {Body: []byte("x"), Hash: hashString("x")},
|
||||||
@@ -1838,10 +1977,11 @@ func intPtr(v int) *int {
|
|||||||
func newMinimalRunner(promptRepo *fakePromptRepo, execRepo *fakeExecutionProfileRepo) *Runner {
|
func newMinimalRunner(promptRepo *fakePromptRepo, execRepo *fakeExecutionProfileRepo) *Runner {
|
||||||
return NewRunner(
|
return NewRunner(
|
||||||
promptRepo,
|
promptRepo,
|
||||||
execRepo,
|
execRepo, nil,
|
||||||
|
|
||||||
defaultArtifactReader(),
|
defaultArtifactReader(),
|
||||||
defaultRenderer(),
|
defaultRenderer(),
|
||||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
|
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
|
||||||
nil,
|
nil)
|
||||||
)
|
|
||||||
}
|
}
|
||||||
|
|||||||
5
json.go
5
json.go
@@ -26,6 +26,7 @@ func (r PreparedRun) MarshalJSON() ([]byte, error) {
|
|||||||
PromptVersion string `json:"prompt_version,omitempty"`
|
PromptVersion string `json:"prompt_version,omitempty"`
|
||||||
PromptHash string `json:"prompt_hash,omitempty"`
|
PromptHash string `json:"prompt_hash,omitempty"`
|
||||||
SelectedProfileID string `json:"selected_profile_id"`
|
SelectedProfileID string `json:"selected_profile_id"`
|
||||||
|
SelectedBackendID string `json:"selected_backend_id,omitempty"`
|
||||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||||
OutputContract OutputContract `json:"output_contract"`
|
OutputContract OutputContract `json:"output_contract"`
|
||||||
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
StructuredOutput *StructuredOutputSpec `json:"structured_output,omitempty"`
|
||||||
@@ -41,6 +42,7 @@ func (r PreparedRun) MarshalJSON() ([]byte, error) {
|
|||||||
PromptVersion: r.PromptVersion,
|
PromptVersion: r.PromptVersion,
|
||||||
PromptHash: r.PromptHash,
|
PromptHash: r.PromptHash,
|
||||||
SelectedProfileID: r.SelectedProfileID,
|
SelectedProfileID: r.SelectedProfileID,
|
||||||
|
SelectedBackendID: r.SelectedBackendID,
|
||||||
EffectiveModelParams: r.EffectiveModelParams,
|
EffectiveModelParams: r.EffectiveModelParams,
|
||||||
OutputContract: r.OutputContract,
|
OutputContract: r.OutputContract,
|
||||||
StructuredOutput: r.StructuredOutput,
|
StructuredOutput: r.StructuredOutput,
|
||||||
@@ -81,6 +83,7 @@ func (r RunResult) MarshalJSON() ([]byte, error) {
|
|||||||
PromptHash: r.PromptHash,
|
PromptHash: r.PromptHash,
|
||||||
RenderedPromptHash: r.RenderedPromptHash,
|
RenderedPromptHash: r.RenderedPromptHash,
|
||||||
SelectedProfileID: r.SelectedProfileID,
|
SelectedProfileID: r.SelectedProfileID,
|
||||||
|
SelectedBackendID: r.SelectedBackendID,
|
||||||
ModelName: r.ModelName,
|
ModelName: r.ModelName,
|
||||||
Endpoint: r.Endpoint,
|
Endpoint: r.Endpoint,
|
||||||
EffectiveModelParams: r.EffectiveModelParams,
|
EffectiveModelParams: r.EffectiveModelParams,
|
||||||
@@ -110,6 +113,7 @@ func (r *RunResult) UnmarshalJSON(data []byte) error {
|
|||||||
PromptHash: wire.PromptHash,
|
PromptHash: wire.PromptHash,
|
||||||
RenderedPromptHash: wire.RenderedPromptHash,
|
RenderedPromptHash: wire.RenderedPromptHash,
|
||||||
SelectedProfileID: wire.SelectedProfileID,
|
SelectedProfileID: wire.SelectedProfileID,
|
||||||
|
SelectedBackendID: wire.SelectedBackendID,
|
||||||
ModelName: wire.ModelName,
|
ModelName: wire.ModelName,
|
||||||
Endpoint: wire.Endpoint,
|
Endpoint: wire.Endpoint,
|
||||||
EffectiveModelParams: wire.EffectiveModelParams,
|
EffectiveModelParams: wire.EffectiveModelParams,
|
||||||
@@ -136,6 +140,7 @@ type runResultJSON struct {
|
|||||||
PromptHash string `json:"prompt_hash,omitempty"`
|
PromptHash string `json:"prompt_hash,omitempty"`
|
||||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||||
SelectedProfileID string `json:"selected_profile_id"`
|
SelectedProfileID string `json:"selected_profile_id"`
|
||||||
|
SelectedBackendID string `json:"selected_backend_id,omitempty"`
|
||||||
ModelName string `json:"model_name"`
|
ModelName string `json:"model_name"`
|
||||||
Endpoint string `json:"endpoint"`
|
Endpoint string `json:"endpoint"`
|
||||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||||
|
|||||||
13
profiles.go
13
profiles.go
@@ -6,16 +6,21 @@ import (
|
|||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// BackendOpenRouter identifies Promptkit's built-in OpenRouter backend.
|
||||||
|
const BackendOpenRouter = backend.OpenRouterID
|
||||||
|
|
||||||
// OpenAICompatibleProfile returns an ordinary in-memory Profile for an
|
// OpenAICompatibleProfile returns an ordinary in-memory Profile for an
|
||||||
// OpenAI-compatible chat-completions endpoint.
|
// OpenAI-compatible chat-completions endpoint.
|
||||||
//
|
//
|
||||||
// It does not register global state, maintain a model catalog, or resolve
|
// It does not register global state, maintain a model catalog, or resolve
|
||||||
// credentials. If APIKeyRequired is true, callers satisfy it with
|
// credentials. If APIKeyRequired is true, callers satisfy it with
|
||||||
// RunRequest.APIKey. Raw API keys do not belong in profiles.
|
// RunRequest.APIKey or an explicit request ExecutionTargetOverride.APIKeyEnv.
|
||||||
|
// Raw API keys do not belong in profiles.
|
||||||
//
|
//
|
||||||
// The function copies the ExtraParams map itself but does not recursively copy
|
// The function copies the ExtraParams map itself but does not recursively copy
|
||||||
// nested values. Validation and a deep copy occur when NewEngine applies a
|
// nested values. Validation and a deep copy occur when NewEngine applies a
|
||||||
@@ -23,6 +28,7 @@ import (
|
|||||||
func OpenAICompatibleProfile(cfg OpenAICompatibleProfileConfig) Profile {
|
func OpenAICompatibleProfile(cfg OpenAICompatibleProfileConfig) Profile {
|
||||||
return Profile{
|
return Profile{
|
||||||
ID: cfg.ID,
|
ID: cfg.ID,
|
||||||
|
BackendID: cfg.BackendID,
|
||||||
Endpoint: cfg.Endpoint,
|
Endpoint: cfg.Endpoint,
|
||||||
Model: cfg.Model,
|
Model: cfg.Model,
|
||||||
Temperature: cfg.Temperature,
|
Temperature: cfg.Temperature,
|
||||||
@@ -85,6 +91,7 @@ func toDomainProfile(publicProfile Profile) (domain.ExecutionProfile, error) {
|
|||||||
}
|
}
|
||||||
prof := domain.ExecutionProfile{
|
prof := domain.ExecutionProfile{
|
||||||
ID: strings.TrimSpace(publicProfile.ID),
|
ID: strings.TrimSpace(publicProfile.ID),
|
||||||
|
BackendID: strings.TrimSpace(publicProfile.BackendID),
|
||||||
Endpoint: publicProfile.Endpoint,
|
Endpoint: publicProfile.Endpoint,
|
||||||
Model: publicProfile.Model,
|
Model: publicProfile.Model,
|
||||||
Temperature: publicProfile.Temperature,
|
Temperature: publicProfile.Temperature,
|
||||||
@@ -106,8 +113,8 @@ func validatePublicProfile(prof domain.ExecutionProfile) error {
|
|||||||
if strings.TrimSpace(prof.ID) == "" {
|
if strings.TrimSpace(prof.ID) == "" {
|
||||||
return errors.New("id is required")
|
return errors.New("id is required")
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(prof.Endpoint) == "" {
|
if strings.TrimSpace(prof.BackendID) == "" && strings.TrimSpace(prof.Endpoint) == "" {
|
||||||
return errors.New("endpoint is required")
|
return errors.New("backend or endpoint is required")
|
||||||
}
|
}
|
||||||
if strings.TrimSpace(prof.Model) == "" {
|
if strings.TrimSpace(prof.Model) == "" {
|
||||||
return errors.New("model is required")
|
return errors.New("model is required")
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ package promptkit_test
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -26,6 +27,62 @@ func TestPreparedRunJSONOmitsZeroTimingValues(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestBackendIdentityJSONNamesAndOmission(t *testing.T) {
|
||||||
|
values := []struct {
|
||||||
|
name string
|
||||||
|
value any
|
||||||
|
field string
|
||||||
|
}{
|
||||||
|
{name: "execution target", value: promptkit.ExecutionTarget{BackendID: promptkit.BackendOpenRouter}, field: "backend_id"},
|
||||||
|
{name: "prepared run", value: promptkit.PreparedRun{SelectedBackendID: promptkit.BackendOpenRouter}, field: "selected_backend_id"},
|
||||||
|
{name: "run result", value: promptkit.RunResult{SelectedBackendID: promptkit.BackendOpenRouter}, field: "selected_backend_id"},
|
||||||
|
}
|
||||||
|
for _, tt := range values {
|
||||||
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
|
payload, err := json.Marshal(tt.value)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal populated value: %v", err)
|
||||||
|
}
|
||||||
|
var object map[string]any
|
||||||
|
if err := json.Unmarshal(payload, &object); err != nil {
|
||||||
|
t.Fatalf("decode populated value: %v", err)
|
||||||
|
}
|
||||||
|
if object[tt.field] != promptkit.BackendOpenRouter {
|
||||||
|
t.Fatalf("expected %s=%q, got %s", tt.field, promptkit.BackendOpenRouter, payload)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
emptyValues := []any{promptkit.ExecutionTarget{}, promptkit.PreparedRun{}, promptkit.RunResult{}}
|
||||||
|
for _, value := range emptyValues {
|
||||||
|
payload, err := json.Marshal(value)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("marshal empty value: %v", err)
|
||||||
|
}
|
||||||
|
if strings.Contains(string(payload), `"backend_id"`) || strings.Contains(string(payload), `"selected_backend_id"`) {
|
||||||
|
t.Fatalf("empty backend identity was not omitted: %s", payload)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestUnknownProfileBackendHasProfileLoadIdentity(t *testing.T) {
|
||||||
|
engine, err := promptkit.NewEngine(promptkit.Config{},
|
||||||
|
promptkit.WithPromptFS(contractPromptFS("prompt", "profile", "message"), "."),
|
||||||
|
promptkit.WithProfiles(promptkit.Profile{ID: "profile", BackendID: "unknown", Model: "model"}),
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("construct engine: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err = engine.Prepare(context.Background(), promptkit.RunRequest{PromptID: "prompt"})
|
||||||
|
if !errors.Is(err, promptkit.ErrProfileLoad) {
|
||||||
|
t.Fatalf("expected ErrProfileLoad, got %v", err)
|
||||||
|
}
|
||||||
|
if errors.Is(err, promptkit.ErrInvalidRequest) {
|
||||||
|
t.Fatalf("unknown backend should not have invalid-request identity: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestPreparedRunJSONTimingRoundTrips(t *testing.T) {
|
func TestPreparedRunJSONTimingRoundTrips(t *testing.T) {
|
||||||
start := time.Date(2026, time.July, 29, 12, 0, 0, 0, time.UTC)
|
start := time.Date(2026, time.July, 29, 12, 0, 0, 0, time.UTC)
|
||||||
prepared := promptkit.PreparedRun{
|
prepared := promptkit.PreparedRun{
|
||||||
|
|||||||
38
types.go
38
types.go
@@ -126,8 +126,12 @@ type PreparedRun struct {
|
|||||||
// SelectedProfileID is the explicit request profile or prompt default that
|
// SelectedProfileID is the explicit request profile or prompt default that
|
||||||
// supplied execution settings.
|
// supplied execution settings.
|
||||||
SelectedProfileID string `json:"selected_profile_id"`
|
SelectedProfileID string `json:"selected_profile_id"`
|
||||||
|
// SelectedBackendID is the selected profile's normalized backend ID. It is
|
||||||
|
// empty for an endpoint-only profile.
|
||||||
|
SelectedBackendID string `json:"selected_backend_id,omitempty"`
|
||||||
// EffectiveModelParams contains framework defaults overlaid by the selected
|
// EffectiveModelParams contains framework defaults overlaid by the selected
|
||||||
// profile and then request overrides. It excludes resolved API-key values.
|
// backend, profile, and then request overrides. It excludes resolved API-key
|
||||||
|
// values.
|
||||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||||
// OutputContract is the complete effective output contract.
|
// OutputContract is the complete effective output contract.
|
||||||
OutputContract OutputContract `json:"output_contract"`
|
OutputContract OutputContract `json:"output_contract"`
|
||||||
@@ -180,6 +184,9 @@ type RunResult struct {
|
|||||||
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
RenderedPromptHash string `json:"rendered_prompt_hash"`
|
||||||
// SelectedProfileID identifies the profile used for execution.
|
// SelectedProfileID identifies the profile used for execution.
|
||||||
SelectedProfileID string `json:"selected_profile_id"`
|
SelectedProfileID string `json:"selected_profile_id"`
|
||||||
|
// SelectedBackendID is the selected profile's normalized backend ID. It is
|
||||||
|
// empty for an endpoint-only profile.
|
||||||
|
SelectedBackendID string `json:"selected_backend_id,omitempty"`
|
||||||
// ModelName is the effective model name and equals
|
// ModelName is the effective model name and equals
|
||||||
// EffectiveModelParams.Model.
|
// EffectiveModelParams.Model.
|
||||||
ModelName string `json:"model_name"`
|
ModelName string `json:"model_name"`
|
||||||
@@ -259,6 +266,10 @@ type ArtifactReader interface {
|
|||||||
// ExecutionTarget represents effective model runtime settings and has a stable
|
// ExecutionTarget represents effective model runtime settings and has a stable
|
||||||
// JSON representation. It never exposes a resolved API-key value.
|
// JSON representation. It never exposes a resolved API-key value.
|
||||||
type ExecutionTarget struct {
|
type ExecutionTarget struct {
|
||||||
|
// BackendID is the effective routing identity selected by the profile. It
|
||||||
|
// remains unchanged when a profile or request overrides Endpoint and is
|
||||||
|
// empty for endpoint-only profiles.
|
||||||
|
BackendID string `json:"backend_id,omitempty"`
|
||||||
// Endpoint is the model-provider base URL.
|
// Endpoint is the model-provider base URL.
|
||||||
Endpoint string `json:"endpoint"`
|
Endpoint string `json:"endpoint"`
|
||||||
// Model is the provider model identifier.
|
// Model is the provider model identifier.
|
||||||
@@ -323,9 +334,10 @@ type ExecutionTargetOverride struct {
|
|||||||
// Profile is an in-memory execution profile for library consumers.
|
// Profile is an in-memory execution profile for library consumers.
|
||||||
//
|
//
|
||||||
// It is equivalent to a loaded profile file after validation. Raw API keys do
|
// It is equivalent to a loaded profile file after validation. Raw API keys do
|
||||||
// not belong in profiles; use APIKeyRequired to require callers to provide
|
// not belong in profiles; use APIKeyRequired to require callers to provide a
|
||||||
// RunRequest.APIKey for each request, or use profile YAML api_key_env with file
|
// RunRequest.APIKey or explicit request ExecutionTargetOverride.APIKeyEnv, or
|
||||||
// and FS profile sources. Profile has no stable JSON representation.
|
// use profile YAML api_key_env with file and FS profile sources. Profile has no
|
||||||
|
// stable JSON representation.
|
||||||
//
|
//
|
||||||
// WithProfiles validates and copies Profile values during NewEngine. Numeric
|
// WithProfiles validates and copies Profile values during NewEngine. Numeric
|
||||||
// zero, blank strings, and an empty ExtraParams map inherit framework defaults;
|
// zero, blank strings, and an empty ExtraParams map inherit framework defaults;
|
||||||
@@ -333,7 +345,12 @@ type ExecutionTargetOverride struct {
|
|||||||
type Profile struct {
|
type Profile struct {
|
||||||
// ID is the required non-blank profile identifier. WithProfiles trims it.
|
// ID is the required non-blank profile identifier. WithProfiles trims it.
|
||||||
ID string
|
ID string
|
||||||
// Endpoint is the required non-blank model-provider base URL.
|
// BackendID optionally selects an engine backend. WithProfiles trims it.
|
||||||
|
// Backend membership is checked when a request selects the profile.
|
||||||
|
BackendID string
|
||||||
|
// Endpoint is the model-provider base URL. It is required only when
|
||||||
|
// BackendID is blank and otherwise overrides the backend endpoint when
|
||||||
|
// non-blank.
|
||||||
Endpoint string
|
Endpoint string
|
||||||
// Model is the required non-blank provider model identifier.
|
// Model is the required non-blank provider model identifier.
|
||||||
Model string
|
Model string
|
||||||
@@ -351,8 +368,9 @@ type Profile struct {
|
|||||||
// ReasoningEffort is optional; a blank value inherits the framework
|
// ReasoningEffort is optional; a blank value inherits the framework
|
||||||
// default.
|
// default.
|
||||||
ReasoningEffort string
|
ReasoningEffort string
|
||||||
// APIKeyRequired requires a non-blank RunRequest.APIKey. It does not store a
|
// APIKeyRequired clears a backend's inherited API-key environment name and
|
||||||
// credential or enable environment lookup.
|
// requires a non-blank RunRequest.APIKey unless the request explicitly
|
||||||
|
// supplies ExecutionTargetOverride.APIKeyEnv. It does not store a credential.
|
||||||
APIKeyRequired bool
|
APIKeyRequired bool
|
||||||
// ExtraParams contains provider-specific JSON-compatible values. An empty
|
// ExtraParams contains provider-specific JSON-compatible values. An empty
|
||||||
// map inherits framework defaults. WithProfiles validates and deeply copies
|
// map inherits framework defaults. WithProfiles validates and deeply copies
|
||||||
@@ -364,13 +382,15 @@ type Profile struct {
|
|||||||
// profile.
|
// profile.
|
||||||
//
|
//
|
||||||
// It contains ordinary profile fields for OpenAI-compatible chat-completions
|
// It contains ordinary profile fields for OpenAI-compatible chat-completions
|
||||||
// endpoints. APIKeyRequired is satisfied by RunRequest.APIKey. Raw API keys do
|
// endpoints. APIKeyRequired follows Profile.APIKeyRequired. Raw API keys do not
|
||||||
// not belong in this config. OpenAICompatibleProfileConfig has no stable JSON
|
// belong in this config. OpenAICompatibleProfileConfig has no stable JSON
|
||||||
// representation and is not validated until its resulting Profile is supplied
|
// representation and is not validated until its resulting Profile is supplied
|
||||||
// through WithProfiles to NewEngine.
|
// through WithProfiles to NewEngine.
|
||||||
type OpenAICompatibleProfileConfig struct {
|
type OpenAICompatibleProfileConfig struct {
|
||||||
// ID becomes Profile.ID.
|
// ID becomes Profile.ID.
|
||||||
ID string
|
ID string
|
||||||
|
// BackendID becomes Profile.BackendID.
|
||||||
|
BackendID string
|
||||||
// Endpoint becomes Profile.Endpoint.
|
// Endpoint becomes Profile.Endpoint.
|
||||||
Endpoint string
|
Endpoint string
|
||||||
// Model becomes Profile.Model.
|
// Model becomes Profile.Model.
|
||||||
|
|||||||
Reference in New Issue
Block a user