Resolve profiles through built-in backends

This commit is contained in:
2026-07-29 17:10:34 +00:00
parent 8d00354c59
commit 810f80e7c9
41 changed files with 538 additions and 173 deletions

View File

@@ -81,6 +81,7 @@ type RunResult struct {
PromptHash string
RenderedPromptHash string
SelectedProfileID string
SelectedBackendID string
ModelName string
Endpoint string
EffectiveModelParams ExecutionTarget
@@ -98,6 +99,7 @@ type PreparedRun struct {
PromptVersion string `json:"prompt_version,omitempty"`
PromptHash string `json:"prompt_hash,omitempty"`
SelectedProfileID string `json:"selected_profile_id"`
SelectedBackendID string `json:"selected_backend_id,omitempty"`
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
TargetPresence ExecutionTargetPresence `json:"-"`
OutputContract OutputContract `json:"output_contract"`
@@ -168,6 +170,7 @@ type Backend struct {
// ExecutionProfile describes how and where to execute a model.
type ExecutionProfile struct {
ID string `yaml:"id"`
BackendID string `yaml:"backend"`
Endpoint string `yaml:"endpoint"`
Model string `yaml:"model"`
Temperature float64 `yaml:"temperature"`
@@ -206,6 +209,7 @@ type ExecutionTargetPresence struct {
// ExecutionTarget represents effective model runtime settings for a run.
type ExecutionTarget struct {
BackendID string `yaml:"backend" json:"backend_id,omitempty"`
Endpoint string `yaml:"endpoint" json:"endpoint"`
Model string `yaml:"model" json:"model"`
Temperature float64 `yaml:"temperature" json:"temperature"`

View File

@@ -1,9 +1,8 @@
id: aion-2
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: aion-labs/aion-2.0
temperature: 0.72
reasoning_effort: high
top_p: 0.95
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,7 +1,6 @@
id: claude-fable-latest
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "~anthropic/claude-fable-latest"
reasoning_effort: high
timeout_seconds: 600
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,7 +1,6 @@
id: claude-haiku-latest
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "~anthropic/claude-haiku-latest"
reasoning_effort: medium
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,7 +1,6 @@
id: claude-opus-latest
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "~anthropic/claude-opus-latest"
reasoning_effort: high
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,7 +1,6 @@
id: claude-sonnet-latest
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "~anthropic/claude-sonnet-latest"
reasoning_effort: high
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,7 +1,6 @@
id: deepseek-3-2
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: deepseek/deepseek-v3.2
reasoning_effort: high
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,7 +1,6 @@
id: deepseek-4-flash
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: deepseek/deepseek-v4-flash
#reasoning_effort: medium
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,7 +1,6 @@
id: deepseek-4-pro
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: deepseek/deepseek-v4-pro
reasoning_effort: high
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,9 +1,8 @@
id: gemini-2-flash-lite
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "google/gemini-2.5-flash-lite"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,9 +1,8 @@
id: gemini-2-flash
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "google/gemini-2.5-flash"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,9 +1,8 @@
id: gemini-2-pro
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "google/gemini-2.5-pro"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,9 +1,8 @@
id: gemini-3-flash-lite
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "google/gemini-3.1-flash-lite"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,9 +1,8 @@
id: gemini-flash-latest
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "~google/gemini-flash-latest"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,9 +1,8 @@
id: gemini-pro-latest
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "~google/gemini-pro-latest"
#temperature: 0.15
reasoning_effort: high
#top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,9 +1,8 @@
id: gemma-4-31b
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: google/gemma-4-31b-it:exacto
temperature: 0.15
reasoning_effort: high
top_p: 0.98
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,9 +1,8 @@
id: minimax-m2
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: minimax/minimax-m2.5
temperature: 0.5
reasoning_effort: high
top_p: 0.95
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,9 +1,8 @@
id: minimax-m3
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: minimax/minimax-m3
#temperature: 0.5
reasoning_effort: high
#top_p: 0.95
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,7 +1,6 @@
id: mistral-large-2512
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: mistralai/mistral-large-2512
temperature: 0.15
top_p: 0.98
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY

View File

@@ -1,8 +1,7 @@
id: mistral-medium-3-5
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: mistralai/mistral-medium-3-5
temperature: 0.15
reasoning_effort: high
top_p: 0.98
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY

View File

@@ -1,7 +1,6 @@
id: mistral-small-3
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: mistralai/mistral-small-3.2-24b-instruct
temperature: 0.05
top_p: 1.0
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY

View File

@@ -1,8 +1,7 @@
id: mistral-small-4
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: mistralai/mistral-small-2603
temperature: 0.1
reasoning_effort: high
top_p: 0.98
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY

View File

@@ -1,7 +1,6 @@
id: nemotron-3-ultra
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: nvidia/nemotron-3-ultra-550b-a55b
reasoning_effort: high
timeout_seconds: 180
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,7 +1,6 @@
id: gpt-5-mini
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "openai/gpt-5.4-mini"
reasoning_effort: high
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -1,7 +1,6 @@
id: gpt-5-nano
endpoint: https://openrouter.ai/api/v1
backend: openrouter
model: "openai/gpt-5.4-nano"
reasoning_effort: high
timeout_seconds: 240
api_key_env: OPENROUTER_API_KEY
service_tier: flex

View File

@@ -7,6 +7,7 @@ import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
"gopkg.in/yaml.v3"
@@ -28,6 +29,12 @@ func TestBuiltInProfilesValidateThroughRepository(t *testing.T) {
if p.ID != id {
t.Fatalf("expected profile id %q, got %q", id, p.ID)
}
if p.BackendID != backend.OpenRouterID {
t.Fatalf("expected profile %q to select %q, got %q", id, backend.OpenRouterID, p.BackendID)
}
if p.Endpoint != "" || p.APIKeyEnv != "" {
t.Fatalf("expected profile %q to inherit backend connection settings, got endpoint=%q api_key_env=%q", id, p.Endpoint, p.APIKeyEnv)
}
})
}
}
@@ -60,6 +67,15 @@ func loadBuiltInProfileIDs(t *testing.T) map[string]string {
if _, ok := raw["api_key"]; ok {
t.Fatalf("built-in profile %s contains raw api_key", name)
}
if raw["backend"] != backend.OpenRouterID {
t.Fatalf("built-in profile %s does not select %q", name, backend.OpenRouterID)
}
if _, ok := raw["endpoint"]; ok {
t.Fatalf("built-in profile %s repeats endpoint", name)
}
if _, ok := raw["api_key_env"]; ok {
t.Fatalf("built-in profile %s repeats api_key_env", name)
}
id, ok := raw["id"].(string)
if !ok || strings.TrimSpace(id) == "" {
t.Fatalf("built-in profile %s has missing id", name)

View File

@@ -121,6 +121,7 @@ func loadProfile(ctx context.Context, fsys fs.FS, root string, id string) (*doma
if prof.ID != id {
continue
}
prof.BackendID = strings.TrimSpace(prof.BackendID)
if err := validateProfile(&prof); err != nil {
if errors.Is(err, ErrRawAPIKeyNotAllowed) {
return nil, fmt.Errorf("%w: %s", err, relPath)
@@ -189,8 +190,8 @@ func validateProfile(p *domain.ExecutionProfile) error {
if strings.TrimSpace(p.ID) == "" {
return errors.New("id is required")
}
if strings.TrimSpace(p.Endpoint) == "" {
return errors.New("endpoint is required")
if strings.TrimSpace(p.BackendID) == "" && strings.TrimSpace(p.Endpoint) == "" {
return errors.New("backend or endpoint is required")
}
if strings.TrimSpace(p.Model) == "" {
return errors.New("model is required")

View File

@@ -52,6 +52,43 @@ func TestFilesystemRepository_GetProfile(t *testing.T) {
}
})
t.Run("backend and endpoint connection matrix", func(t *testing.T) {
tests := []struct {
name string
connection string
wantBackend string
wantEndpoint string
wantErr bool
}{
{name: "backend only", connection: "backend: ' openrouter '", wantBackend: "openrouter"},
{name: "endpoint only", connection: "endpoint: http://localhost:8000/v1", wantEndpoint: "http://localhost:8000/v1"},
{name: "both", connection: "backend: openrouter\nendpoint: http://localhost:8000/v1", wantBackend: "openrouter", wantEndpoint: "http://localhost:8000/v1"},
{name: "neither", wantErr: true},
{name: "blank backend", connection: "backend: ' '", wantErr: true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
id := "connection-" + strings.ReplaceAll(tt.name, " ", "-")
writeProfileTestFile(t, filepath.Join(tmpDir, id+".yaml"), "id: "+id+"\nmodel: model\n"+tt.connection+"\n")
p, err := repo.GetProfile(ctx, id)
if tt.wantErr {
if !errors.Is(err, ErrInvalidProfile) {
t.Fatalf("expected ErrInvalidProfile, got %v", err)
}
return
}
if err != nil {
t.Fatalf("expected profile to load, got %v", err)
}
if p.BackendID != tt.wantBackend || p.Endpoint != tt.wantEndpoint {
t.Fatalf("unexpected connection values: backend=%q endpoint=%q", p.BackendID, p.Endpoint)
}
})
}
})
t.Run("valid profile with api_key_env", func(t *testing.T) {
p, err := repo.GetProfile(ctx, "local-secure")
if err != nil {

View File

@@ -40,6 +40,7 @@ var (
type Runner struct {
promptDefs promptdef.Repository
profiles profile.Repository
backends BackendResolver
artifacts artifact.Reader
renderer prompt.Renderer
llm llm.Client
@@ -47,20 +48,27 @@ type Runner struct {
repairer OutputRepairer
}
// BackendResolver resolves one normalized backend ID.
type BackendResolver interface {
GetBackend(string) (domain.Backend, error)
}
func NewRunner(
promptDefs promptdef.Repository,
profiles profile.Repository,
backends BackendResolver,
artifacts artifact.Reader,
renderer prompt.Renderer,
llmClient llm.Client,
validator validate.Validator,
) *Runner {
return NewRunnerWithRepairer(promptDefs, profiles, artifacts, renderer, llmClient, validator, nil)
return NewRunnerWithRepairer(promptDefs, profiles, backends, artifacts, renderer, llmClient, validator, nil)
}
func NewRunnerWithRepairer(
promptDefs promptdef.Repository,
profiles profile.Repository,
backends BackendResolver,
artifacts artifact.Reader,
renderer prompt.Renderer,
llmClient llm.Client,
@@ -70,6 +78,7 @@ func NewRunnerWithRepairer(
return &Runner{
promptDefs: promptDefs,
profiles: profiles,
backends: backends,
artifacts: artifacts,
renderer: renderer,
llm: llmClient,
@@ -153,6 +162,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
PromptHash: prepared.PromptHash,
RenderedPromptHash: prepared.RenderedPromptHash,
SelectedProfileID: prepared.SelectedProfileID,
SelectedBackendID: prepared.SelectedBackendID,
ModelName: prepared.EffectiveModelParams.Model,
Endpoint: prepared.EffectiveModelParams.Endpoint,
EffectiveModelParams: prepared.EffectiveModelParams,
@@ -193,7 +203,20 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
}
effectiveModel, targetPresence, err := resolveExecutionTarget(execProfile, req.Execution)
var selectedBackend *domain.Backend
if backendID := strings.TrimSpace(execProfile.BackendID); backendID != "" {
execProfile.BackendID = backendID
if r.backends == nil {
return nil, fmt.Errorf("%w: backend %q cannot be resolved", ErrProfileLoad, backendID)
}
resolvedBackend, resolveErr := r.backends.GetBackend(backendID)
if resolveErr != nil {
return nil, fmt.Errorf("%w: backend %q: %w", ErrProfileLoad, backendID, resolveErr)
}
selectedBackend = &resolvedBackend
}
effectiveModel, targetPresence, err := resolveExecutionTarget(selectedBackend, execProfile, req.Execution)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
}
@@ -239,6 +262,7 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
PromptVersion: def.Version,
PromptHash: promptDefinitionHash,
SelectedProfileID: selectedProfileID,
SelectedBackendID: effectiveModel.BackendID,
EffectiveModelParams: effectiveModel,
TargetPresence: targetPresence,
OutputContract: effectiveContract,
@@ -338,6 +362,9 @@ func (r *Runner) shouldAttemptRepair(contract domain.OutputContract, validationR
func mergeExecutionTarget(base domain.ExecutionTarget, override domain.ExecutionTarget) domain.ExecutionTarget {
out := base
if strings.TrimSpace(override.BackendID) != "" {
out.BackendID = override.BackendID
}
if override.Endpoint != "" {
out.Endpoint = override.Endpoint
}
@@ -367,6 +394,7 @@ func mergeExecutionTarget(base domain.ExecutionTarget, override domain.Execution
}
if override.APIKeyRequired {
out.APIKeyRequired = true
out.APIKeyEnv = ""
}
if len(override.ExtraParams) > 0 {
out.ExtraParams = copyExtraParams(override.ExtraParams)
@@ -426,8 +454,9 @@ func mergeExecutionTargetOverride(base domain.ExecutionTarget, override domain.E
return out, presence, nil
}
func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) {
func resolveExecutionTarget(backendValue *domain.Backend, profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) {
out := defaults.ExecutionTargetDefault()
out = mergeExecutionTarget(out, backendToTarget(backendValue))
out = mergeExecutionTarget(out, executionProfileToTarget(profileValue))
var presence domain.ExecutionTargetPresence
if override != nil {
@@ -461,8 +490,13 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
if p == nil {
return domain.ExecutionTarget{}
}
endpoint := p.Endpoint
if strings.TrimSpace(endpoint) == "" {
endpoint = ""
}
return domain.ExecutionTarget{
Endpoint: p.Endpoint,
BackendID: p.BackendID,
Endpoint: endpoint,
Model: p.Model,
Temperature: p.Temperature,
MaxTokens: p.MaxTokens,
@@ -476,6 +510,18 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
}
}
func backendToTarget(value *domain.Backend) domain.ExecutionTarget {
if value == nil {
return domain.ExecutionTarget{}
}
return domain.ExecutionTarget{
BackendID: value.ID,
Endpoint: value.Endpoint,
APIKeyEnv: value.APIKeyEnv,
ExtraParams: copyExtraParams(value.ExtraParams),
}
}
func copyExtraParams(src map[string]any) map[string]any {
if len(src) == 0 {
return nil

View File

@@ -34,6 +34,18 @@ type fakeExecutionProfileRepo struct {
lastID string
}
type fakeBackendResolver struct {
backends map[string]domain.Backend
}
func (f fakeBackendResolver) GetBackend(id string) (domain.Backend, error) {
value, ok := f.backends[id]
if !ok {
return domain.Backend{}, errors.New("backend not found")
}
return value, nil
}
func (f *fakeExecutionProfileRepo) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
f.lastID = id
if f.err != nil {
@@ -167,7 +179,7 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) {
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
llmClient := &fakeLLM{forbid: true}
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
PromptVersion: "1",
@@ -257,7 +269,7 @@ func TestRunnerPrepareSelectedProfileDoesNotExistFails(t *testing.T) {
}
func TestRunnerPreparePromptLoadFailure(t *testing.T) {
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p"})
if !errors.Is(err, ErrPromptLoad) {
t.Fatalf("expected ErrPromptLoad, got %v", err)
@@ -281,7 +293,7 @@ func TestRunnerPrepareRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) {
ServiceTier: "priority",
},
}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -379,12 +391,12 @@ func TestRunnerPrepareRequestNumericOverridePresence(t *testing.T) {
TopP: 0.8,
TimeoutSeconds: 45,
},
}},
}}, nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{forbid: true},
nil,
)
nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -426,12 +438,12 @@ func TestRunnerPrepareInvalidRequestNumericOverridesFail(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{forbid: true},
nil,
)
nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -458,7 +470,7 @@ func TestRunnerPrepareSelectedProfileBeatsBuiltInDefault(t *testing.T) {
ServiceTier: "priority",
},
}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -495,12 +507,12 @@ func TestRunnerPrepareFileBackedPromptBodiesRenderCorrectly(t *testing.T) {
llmClient := &fakeLLM{forbid: true}
runner := NewRunner(
promptdef.NewFilesystemRepository(promptDir),
profile.NewFilesystemRepository(profileDir),
profile.NewFilesystemRepository(profileDir), nil,
reader,
prompt.NewGoRenderer(),
llmClient,
nil,
)
nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "valid-file-backed",
@@ -526,12 +538,13 @@ func TestRunnerPrepareRequiredInputMissingFails(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
prompt.NewGoRenderer(),
&fakeLLM{forbid: true},
nil,
)
nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
@@ -552,12 +565,13 @@ func TestRunnerPrepareUnknownTemplateInputReferenceFails(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
prompt.NewGoRenderer(),
&fakeLLM{forbid: true},
nil,
)
nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
@@ -579,7 +593,7 @@ func TestRunnerPrepareAPIKeyEnvNameIncludedButNotResolvedValue(t *testing.T) {
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName},
}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if err != nil {
@@ -607,12 +621,12 @@ func TestRunnerPrepareJSONSchemaBuildsStructuredOutputSpec(t *testing.T) {
}
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{forbid: true},
validator,
)
validator)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -651,12 +665,12 @@ func TestRunnerPrepareJSONSchemaSchemaLoadFailureReturnsValidationError(t *testi
validator := &fakeValidator{schemaErr: errors.New("schema unavailable")}
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{forbid: true},
validator,
)
validator)
_, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -681,12 +695,12 @@ func TestRunnerRunJSONSchemaSchemaLoadFailureFailsBeforeLLM(t *testing.T) {
validator := &fakeValidator{schemaErr: errors.New("schema unavailable")}
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
defaultRenderer(),
llmClient,
validator,
)
validator)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -836,7 +850,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}}
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
PromptVersion: "1",
@@ -915,7 +929,7 @@ func TestRunnerRunPassesExtraParamsToGenerateRequestTarget(t *testing.T) {
},
}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -941,7 +955,7 @@ func TestRunnerRunAndPrepareResolveSameProfileAndEffectiveSettings(t *testing.T)
}}
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap"}}
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil)
req := domain.RunRequest{
PromptID: "p",
@@ -1071,7 +1085,7 @@ func TestRunnerRunExplicitRuntimeOverrideBeatsSelectedProfileValue(t *testing.T)
},
}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1115,7 +1129,7 @@ func TestRunnerRunSelectedProfileBeatsBuiltInDefault(t *testing.T) {
},
}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1142,7 +1156,7 @@ func TestRunnerRunBuiltInDefaultsUsedWhenProfileOmitsOptionalFields(t *testing.T
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"},
}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1172,7 +1186,7 @@ func TestRunnerRunAPIKeyEnvResolvesFromEnvironment(t *testing.T) {
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_TEST_API_KEY"},
}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if err != nil {
@@ -1188,7 +1202,7 @@ func TestRunnerRunAPIKeyEnvMissingEnvironmentValueFailsClearly(t *testing.T) {
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_MISSING_KEY"},
}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if !errors.Is(err, ErrInvalidRequest) {
@@ -1209,7 +1223,7 @@ func TestRunnerRunDirectAPIKeyBypassesMissingEnvAndReachesLLM(t *testing.T) {
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_MISSING_KEY"},
}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1233,7 +1247,7 @@ func TestRunnerPrepareAPIKeyRequiredFailsWithoutDirectKey(t *testing.T) {
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true},
}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1252,7 +1266,7 @@ func TestRunnerRunAPIKeyRequiredSucceedsWithDirectKey(t *testing.T) {
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true},
}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1279,7 +1293,7 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideWorks(t *testing.T) {
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"},
}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1304,7 +1318,7 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideBeatsProfile(t *testing.T) {
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: profileEnv},
}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1328,7 +1342,7 @@ func TestRunnerRunAPIKeyValueNotPresentInMetadata(t *testing.T) {
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName},
}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if err != nil {
@@ -1344,7 +1358,7 @@ func TestRunnerRunAPIKeyValueNotPresentInMetadata(t *testing.T) {
}
func TestRunnerRunPromptLoadFailure(t *testing.T) {
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"})
if !errors.Is(err, ErrPromptLoad) {
t.Fatalf("expected ErrPromptLoad, got %v", err)
@@ -1357,12 +1371,12 @@ func TestRunnerRunPromptLoadFailure(t *testing.T) {
func TestRunnerRunArtifactLoadFailure(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
&fakeArtifactReader{errByURI: map[string]error{"a://bad": errors.New("read failed")}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
nil,
)
nil)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1377,12 +1391,13 @@ func TestRunnerRunArtifactLoadFailure(t *testing.T) {
func TestRunnerRunPromptRenderFailure(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
&fakeRenderer{err: errors.New("render failed")},
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
nil,
)
nil)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
@@ -1396,12 +1411,13 @@ func TestRunnerRunPromptRenderFailure(t *testing.T) {
func TestRunnerRunLLMFailure(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{err: errors.New("llm failed")},
nil,
)
nil)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
@@ -1415,12 +1431,13 @@ func TestRunnerRunLLMFailure(t *testing.T) {
func TestRunnerRunCancellationPreservesGenerationCategory(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{resp: &domain.GenerateResponse{Content: "ignored"}},
nil,
)
nil)
ctx, cancel := context.WithCancel(context.Background())
cancel()
@@ -1440,12 +1457,13 @@ func TestRunnerRunCancellationPreservesGenerationCategory(t *testing.T) {
func TestRunnerRunLLMInvalidRequestMapsToUsecaseInvalidRequest(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{err: llm.ErrInvalidRequest},
nil,
)
nil)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
@@ -1463,12 +1481,13 @@ func TestRunnerRunValidationStillWorks(t *testing.T) {
validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}}
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}},
validator,
)
validator)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
@@ -1489,14 +1508,17 @@ func TestRunnerRunStructuredRepairRemainsBoundedAndUsesEffectiveModelSettings(t
runner := NewRunnerWithRepairer(
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", TimeoutSeconds: 55},
"exec": {ID: "exec", BackendID: "custom", Model: "profile-model", TimeoutSeconds: 55},
}}, fakeBackendResolver{backends: map[string]domain.Backend{
"custom": {ID: "custom", Endpoint: "http://backend/v1"},
}},
defaultArtifactReader(),
defaultRenderer(),
llmClient,
validate.NewStandardValidator("."),
repairer,
)
repairer)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
@@ -1518,6 +1540,12 @@ func TestRunnerRunStructuredRepairRemainsBoundedAndUsesEffectiveModelSettings(t
if repairer.reqs[0].Target.TimeoutSeconds != 22 {
t.Fatalf("expected repair to use effective timeout, got %d", repairer.reqs[0].Target.TimeoutSeconds)
}
if llmClient.lastReq.Target.BackendID != "custom" ||
repairer.reqs[0].Target.BackendID != "custom" ||
res.SelectedBackendID != "custom" {
t.Fatalf("expected backend identity in generation, repair, and result: generate=%q repair=%q result=%q",
llmClient.lastReq.Target.BackendID, repairer.reqs[0].Target.BackendID, res.SelectedBackendID)
}
}
func TestRunnerRunJSONSchemaRepairCarriesStructuredOutputSpec(t *testing.T) {
@@ -1546,13 +1574,13 @@ func TestRunnerRunJSONSchemaRepairCarriesStructuredOutputSpec(t *testing.T) {
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: `{"events":[1]}`}}
runner := NewRunnerWithRepairer(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
defaultRenderer(),
llmClient,
validator,
repairer,
)
repairer)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1629,13 +1657,12 @@ func TestResolveExecutionTargetProfileValuesPopulateAllSupportedFields(t *testin
ServiceTier: "priority",
ReasoningEffort: "low",
APIKeyEnv: "PROFILE_KEY",
APIKeyRequired: true,
ExtraParams: map[string]any{
"profile_option": "enabled",
},
}
target, presence, err := resolveExecutionTarget(profileValue, nil)
target, presence, err := resolveExecutionTarget(nil, profileValue, nil)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
@@ -1690,7 +1717,7 @@ func TestResolveExecutionTargetRuntimeOverridesBeatProfileForAllOverrideableFiel
},
}
target, presence, err := resolveExecutionTarget(profileValue, override)
target, presence, err := resolveExecutionTarget(nil, profileValue, override)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
@@ -1813,6 +1840,118 @@ func defaultExecutionProfile() *domain.ExecutionProfile {
}
}
func TestResolveExecutionTargetUsesBackendProfileAndRequestPrecedence(t *testing.T) {
backendValue := &domain.Backend{
ID: "custom",
Endpoint: "http://backend/v1",
APIKeyEnv: "BACKEND_KEY",
ExtraParams: map[string]any{"backend": true},
}
profileValue := &domain.ExecutionProfile{
ID: "exec",
BackendID: "custom",
Endpoint: "http://profile/v1",
Model: "profile-model",
APIKeyEnv: "PROFILE_KEY",
ExtraParams: map[string]any{"profile": true},
}
override := &domain.ExecutionTargetOverride{
Endpoint: "http://request/v1",
APIKeyEnv: "REQUEST_KEY",
ExtraParams: map[string]any{"request": true},
}
target, _, err := resolveExecutionTarget(backendValue, profileValue, override)
if err != nil {
t.Fatalf("resolve target: %v", err)
}
if target.BackendID != "custom" {
t.Fatalf("endpoint override changed backend identity: %+v", target)
}
if target.Endpoint != "http://request/v1" || target.APIKeyEnv != "REQUEST_KEY" {
t.Fatalf("request values did not win: %+v", target)
}
if !reflect.DeepEqual(target.ExtraParams, map[string]any{"request": true}) {
t.Fatalf("expected whole-map request replacement, got %#v", target.ExtraParams)
}
target, _, err = resolveExecutionTarget(backendValue, &domain.ExecutionProfile{
ID: "exec", BackendID: "custom", Model: "profile-model",
}, nil)
if err != nil {
t.Fatalf("resolve backend defaults: %v", err)
}
if target.Endpoint != backendValue.Endpoint ||
target.APIKeyEnv != backendValue.APIKeyEnv ||
!reflect.DeepEqual(target.ExtraParams, backendValue.ExtraParams) {
t.Fatalf("backend defaults were not inherited: %+v", target)
}
}
func TestRunnerPrepareBackendResolutionAndCredentialPrecedence(t *testing.T) {
resolver := fakeBackendResolver{backends: map[string]domain.Backend{
"custom": {ID: "custom", Endpoint: "http://backend/v1", APIKeyEnv: "BACKEND_KEY"},
}}
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
t.Run("unknown backend is a profile load failure", func(t *testing.T) {
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", BackendID: "unknown", Model: "model"},
}}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if !errors.Is(err, ErrProfileLoad) {
t.Fatalf("expected ErrProfileLoad, got %v", err)
}
})
t.Run("nil resolver is a profile load failure", func(t *testing.T) {
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", BackendID: "custom", Model: "model"},
}}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if !errors.Is(err, ErrProfileLoad) {
t.Fatalf("expected ErrProfileLoad, got %v", err)
}
})
t.Run("request environment wins", func(t *testing.T) {
t.Setenv("REQUEST_KEY", "request-secret")
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", BackendID: "custom", Model: "model", APIKeyEnv: "PROFILE_KEY"},
}}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(),
Execution: &domain.ExecutionTargetOverride{APIKeyEnv: "REQUEST_KEY"},
})
if err != nil {
t.Fatalf("prepare: %v", err)
}
if prepared.EffectiveModelParams.APIKeyEnv != "REQUEST_KEY" {
t.Fatalf("unexpected credential source: %+v", prepared.EffectiveModelParams)
}
})
t.Run("required direct key clears inherited environment", func(t *testing.T) {
t.Setenv("BACKEND_KEY", "backend-secret")
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", BackendID: "custom", Model: "model", APIKeyRequired: true},
}}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if !errors.Is(err, ErrAPIKeyRequired) {
t.Fatalf("expected ErrAPIKeyRequired, got %v", err)
}
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(), APIKey: "direct-secret",
})
if err != nil {
t.Fatalf("prepare with direct key: %v", err)
}
if prepared.EffectiveModelParams.APIKeyEnv != "" {
t.Fatalf("expected inherited environment name to be cleared, got %q", prepared.EffectiveModelParams.APIKeyEnv)
}
})
}
func defaultArtifactReader() *fakeArtifactReader {
return &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{
"a://ok": {Body: []byte("x"), Hash: hashString("x")},
@@ -1838,10 +1977,11 @@ func intPtr(v int) *int {
func newMinimalRunner(promptRepo *fakePromptRepo, execRepo *fakeExecutionProfileRepo) *Runner {
return NewRunner(
promptRepo,
execRepo,
execRepo, nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
nil,
)
nil)
}