eric/promptkit
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Resolve profiles through built-in backends

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-29 17:10:34 +00:00
parent 8d00354c59
commit 810f80e7c9
41 changed files with 538 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files

54
internal/usecase/runner.go
View File

@@ -40,6 +40,7 @@ var (
type Runner struct {
promptDefs promptdef.Repository
profiles profile.Repository
backends BackendResolver
artifacts artifact.Reader
renderer prompt.Renderer
llm llm.Client
@@ -47,20 +48,27 @@ type Runner struct {
repairer OutputRepairer
}
// BackendResolver resolves one normalized backend ID.
type BackendResolver interface {
GetBackend(string) (domain.Backend, error)
}
func NewRunner(
promptDefs promptdef.Repository,
profiles profile.Repository,
backends BackendResolver,
artifacts artifact.Reader,
renderer prompt.Renderer,
llmClient llm.Client,
validator validate.Validator,
) *Runner {
return NewRunnerWithRepairer(promptDefs, profiles, artifacts, renderer, llmClient, validator, nil)
return NewRunnerWithRepairer(promptDefs, profiles, backends, artifacts, renderer, llmClient, validator, nil)
}
func NewRunnerWithRepairer(
promptDefs promptdef.Repository,
profiles profile.Repository,
backends BackendResolver,
artifacts artifact.Reader,
renderer prompt.Renderer,
llmClient llm.Client,
@@ -70,6 +78,7 @@ func NewRunnerWithRepairer(
return &Runner{
promptDefs: promptDefs,
profiles: profiles,
backends: backends,
artifacts: artifacts,
renderer: renderer,
llm: llmClient,
@@ -153,6 +162,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
PromptHash: prepared.PromptHash,
RenderedPromptHash: prepared.RenderedPromptHash,
SelectedProfileID: prepared.SelectedProfileID,
SelectedBackendID: prepared.SelectedBackendID,
ModelName: prepared.EffectiveModelParams.Model,
Endpoint: prepared.EffectiveModelParams.Endpoint,
EffectiveModelParams: prepared.EffectiveModelParams,
@@ -193,7 +203,20 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
}
effectiveModel, targetPresence, err := resolveExecutionTarget(execProfile, req.Execution)
var selectedBackend *domain.Backend
if backendID := strings.TrimSpace(execProfile.BackendID); backendID != "" {
execProfile.BackendID = backendID
if r.backends == nil {
return nil, fmt.Errorf("%w: backend %q cannot be resolved", ErrProfileLoad, backendID)
}
resolvedBackend, resolveErr := r.backends.GetBackend(backendID)
if resolveErr != nil {
return nil, fmt.Errorf("%w: backend %q: %w", ErrProfileLoad, backendID, resolveErr)
}
selectedBackend = &resolvedBackend
}
effectiveModel, targetPresence, err := resolveExecutionTarget(selectedBackend, execProfile, req.Execution)
if err != nil {
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
}
@@ -239,6 +262,7 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
PromptVersion: def.Version,
PromptHash: promptDefinitionHash,
SelectedProfileID: selectedProfileID,
SelectedBackendID: effectiveModel.BackendID,
EffectiveModelParams: effectiveModel,
TargetPresence: targetPresence,
OutputContract: effectiveContract,
@@ -338,6 +362,9 @@ func (r *Runner) shouldAttemptRepair(contract domain.OutputContract, validationR
func mergeExecutionTarget(base domain.ExecutionTarget, override domain.ExecutionTarget) domain.ExecutionTarget {
out := base
if strings.TrimSpace(override.BackendID) != "" {
out.BackendID = override.BackendID
}
if override.Endpoint != "" {
out.Endpoint = override.Endpoint
}
@@ -367,6 +394,7 @@ func mergeExecutionTarget(base domain.ExecutionTarget, override domain.Execution
}
if override.APIKeyRequired {
out.APIKeyRequired = true
out.APIKeyEnv = ""
}
if len(override.ExtraParams) > 0 {
out.ExtraParams = copyExtraParams(override.ExtraParams)
@@ -426,8 +454,9 @@ func mergeExecutionTargetOverride(base domain.ExecutionTarget, override domain.E
return out, presence, nil
}
func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) {
func resolveExecutionTarget(backendValue *domain.Backend, profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) {
out := defaults.ExecutionTargetDefault()
out = mergeExecutionTarget(out, backendToTarget(backendValue))
out = mergeExecutionTarget(out, executionProfileToTarget(profileValue))
var presence domain.ExecutionTargetPresence
if override != nil {
@@ -461,8 +490,13 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
if p == nil {
return domain.ExecutionTarget{}
}
endpoint := p.Endpoint
if strings.TrimSpace(endpoint) == "" {
endpoint = ""
}
return domain.ExecutionTarget{
Endpoint: p.Endpoint,
BackendID: p.BackendID,
Endpoint: endpoint,
Model: p.Model,
Temperature: p.Temperature,
MaxTokens: p.MaxTokens,
@@ -476,6 +510,18 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
}
}
func backendToTarget(value *domain.Backend) domain.ExecutionTarget {
if value == nil {
return domain.ExecutionTarget{}
}
return domain.ExecutionTarget{
BackendID: value.ID,
Endpoint: value.Endpoint,
APIKeyEnv: value.APIKeyEnv,
ExtraParams: copyExtraParams(value.ExtraParams),
}
}
func copyExtraParams(src map[string]any) map[string]any {
if len(src) == 0 {
return nil

288
internal/usecase/runner_test.go
View File

@@ -34,6 +34,18 @@ type fakeExecutionProfileRepo struct {
lastID string
}
type fakeBackendResolver struct {
backends map[string]domain.Backend
}
func (f fakeBackendResolver) GetBackend(id string) (domain.Backend, error) {
value, ok := f.backends[id]
if !ok {
return domain.Backend{}, errors.New("backend not found")
}
return value, nil
}
func (f *fakeExecutionProfileRepo) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
f.lastID = id
if f.err != nil {
@@ -167,7 +179,7 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) {
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
llmClient := &fakeLLM{forbid: true}
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
PromptVersion: "1",
@@ -257,7 +269,7 @@ func TestRunnerPrepareSelectedProfileDoesNotExistFails(t *testing.T) {
}
func TestRunnerPreparePromptLoadFailure(t *testing.T) {
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p"})
if !errors.Is(err, ErrPromptLoad) {
t.Fatalf("expected ErrPromptLoad, got %v", err)
@@ -281,7 +293,7 @@ func TestRunnerPrepareRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) {
ServiceTier: "priority",
},
}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -379,12 +391,12 @@ func TestRunnerPrepareRequestNumericOverridePresence(t *testing.T) {
TopP: 0.8,
TimeoutSeconds: 45,
},
}},
}}, nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{forbid: true},
nil,
)
nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -426,12 +438,12 @@ func TestRunnerPrepareInvalidRequestNumericOverridesFail(t *testing.T) {
t.Run(tc.name, func(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{forbid: true},
nil,
)
nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -458,7 +470,7 @@ func TestRunnerPrepareSelectedProfileBeatsBuiltInDefault(t *testing.T) {
ServiceTier: "priority",
},
}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -495,12 +507,12 @@ func TestRunnerPrepareFileBackedPromptBodiesRenderCorrectly(t *testing.T) {
llmClient := &fakeLLM{forbid: true}
runner := NewRunner(
promptdef.NewFilesystemRepository(promptDir),
profile.NewFilesystemRepository(profileDir),
profile.NewFilesystemRepository(profileDir), nil,
reader,
prompt.NewGoRenderer(),
llmClient,
nil,
)
nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "valid-file-backed",
@@ -526,12 +538,13 @@ func TestRunnerPrepareRequiredInputMissingFails(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
prompt.NewGoRenderer(),
&fakeLLM{forbid: true},
nil,
)
nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
@@ -552,12 +565,13 @@ func TestRunnerPrepareUnknownTemplateInputReferenceFails(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
prompt.NewGoRenderer(),
&fakeLLM{forbid: true},
nil,
)
nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
@@ -579,7 +593,7 @@ func TestRunnerPrepareAPIKeyEnvNameIncludedButNotResolvedValue(t *testing.T) {
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName},
}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if err != nil {
@@ -607,12 +621,12 @@ func TestRunnerPrepareJSONSchemaBuildsStructuredOutputSpec(t *testing.T) {
}
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{forbid: true},
validator,
)
validator)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -651,12 +665,12 @@ func TestRunnerPrepareJSONSchemaSchemaLoadFailureReturnsValidationError(t *testi
validator := &fakeValidator{schemaErr: errors.New("schema unavailable")}
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{forbid: true},
validator,
)
validator)
_, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -681,12 +695,12 @@ func TestRunnerRunJSONSchemaSchemaLoadFailureFailsBeforeLLM(t *testing.T) {
validator := &fakeValidator{schemaErr: errors.New("schema unavailable")}
runner := NewRunner(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
defaultRenderer(),
llmClient,
validator,
)
validator)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -836,7 +850,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}}
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
PromptVersion: "1",
@@ -915,7 +929,7 @@ func TestRunnerRunPassesExtraParamsToGenerateRequestTarget(t *testing.T) {
},
}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -941,7 +955,7 @@ func TestRunnerRunAndPrepareResolveSameProfileAndEffectiveSettings(t *testing.T)
}}
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap"}}
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil)
req := domain.RunRequest{
PromptID: "p",
@@ -1071,7 +1085,7 @@ func TestRunnerRunExplicitRuntimeOverrideBeatsSelectedProfileValue(t *testing.T)
},
}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1115,7 +1129,7 @@ func TestRunnerRunSelectedProfileBeatsBuiltInDefault(t *testing.T) {
},
}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1142,7 +1156,7 @@ func TestRunnerRunBuiltInDefaultsUsedWhenProfileOmitsOptionalFields(t *testing.T
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"},
}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1172,7 +1186,7 @@ func TestRunnerRunAPIKeyEnvResolvesFromEnvironment(t *testing.T) {
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_TEST_API_KEY"},
}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if err != nil {
@@ -1188,7 +1202,7 @@ func TestRunnerRunAPIKeyEnvMissingEnvironmentValueFailsClearly(t *testing.T) {
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_MISSING_KEY"},
}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if !errors.Is(err, ErrInvalidRequest) {
@@ -1209,7 +1223,7 @@ func TestRunnerRunDirectAPIKeyBypassesMissingEnvAndReachesLLM(t *testing.T) {
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_MISSING_KEY"},
}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1233,7 +1247,7 @@ func TestRunnerPrepareAPIKeyRequiredFailsWithoutDirectKey(t *testing.T) {
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true},
}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1252,7 +1266,7 @@ func TestRunnerRunAPIKeyRequiredSucceedsWithDirectKey(t *testing.T) {
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true},
}}
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1279,7 +1293,7 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideWorks(t *testing.T) {
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"},
}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1304,7 +1318,7 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideBeatsProfile(t *testing.T) {
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: profileEnv},
}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1328,7 +1342,7 @@ func TestRunnerRunAPIKeyValueNotPresentInMetadata(t *testing.T) {
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName},
}}
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
res, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if err != nil {
@@ -1344,7 +1358,7 @@ func TestRunnerRunAPIKeyValueNotPresentInMetadata(t *testing.T) {
}
func TestRunnerRunPromptLoadFailure(t *testing.T) {
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"})
if !errors.Is(err, ErrPromptLoad) {
t.Fatalf("expected ErrPromptLoad, got %v", err)
@@ -1357,12 +1371,12 @@ func TestRunnerRunPromptLoadFailure(t *testing.T) {
func TestRunnerRunArtifactLoadFailure(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
&fakeArtifactReader{errByURI: map[string]error{"a://bad": errors.New("read failed")}},
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
nil,
)
nil)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1377,12 +1391,13 @@ func TestRunnerRunArtifactLoadFailure(t *testing.T) {
func TestRunnerRunPromptRenderFailure(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
&fakeRenderer{err: errors.New("render failed")},
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
nil,
)
nil)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
@@ -1396,12 +1411,13 @@ func TestRunnerRunPromptRenderFailure(t *testing.T) {
func TestRunnerRunLLMFailure(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{err: errors.New("llm failed")},
nil,
)
nil)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
@@ -1415,12 +1431,13 @@ func TestRunnerRunLLMFailure(t *testing.T) {
func TestRunnerRunCancellationPreservesGenerationCategory(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{resp: &domain.GenerateResponse{Content: "ignored"}},
nil,
)
nil)
ctx, cancel := context.WithCancel(context.Background())
cancel()
@@ -1440,12 +1457,13 @@ func TestRunnerRunCancellationPreservesGenerationCategory(t *testing.T) {
func TestRunnerRunLLMInvalidRequestMapsToUsecaseInvalidRequest(t *testing.T) {
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{err: llm.ErrInvalidRequest},
nil,
)
nil)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
@@ -1463,12 +1481,13 @@ func TestRunnerRunValidationStillWorks(t *testing.T) {
validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}}
runner := NewRunner(
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}},
validator,
)
validator)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
@@ -1489,14 +1508,17 @@ func TestRunnerRunStructuredRepairRemainsBoundedAndUsesEffectiveModelSettings(t
runner := NewRunnerWithRepairer(
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", TimeoutSeconds: 55},
"exec": {ID: "exec", BackendID: "custom", Model: "profile-model", TimeoutSeconds: 55},
}}, fakeBackendResolver{backends: map[string]domain.Backend{
"custom": {ID: "custom", Endpoint: "http://backend/v1"},
}},
defaultArtifactReader(),
defaultRenderer(),
llmClient,
validate.NewStandardValidator("."),
repairer,
)
repairer)
res, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
ProfileID: "exec",
@@ -1518,6 +1540,12 @@ func TestRunnerRunStructuredRepairRemainsBoundedAndUsesEffectiveModelSettings(t
if repairer.reqs[0].Target.TimeoutSeconds != 22 {
t.Fatalf("expected repair to use effective timeout, got %d", repairer.reqs[0].Target.TimeoutSeconds)
}
if llmClient.lastReq.Target.BackendID != "custom" ||
repairer.reqs[0].Target.BackendID != "custom" ||
res.SelectedBackendID != "custom" {
t.Fatalf("expected backend identity in generation, repair, and result: generate=%q repair=%q result=%q",
llmClient.lastReq.Target.BackendID, repairer.reqs[0].Target.BackendID, res.SelectedBackendID)
}
}
func TestRunnerRunJSONSchemaRepairCarriesStructuredOutputSpec(t *testing.T) {
@@ -1546,13 +1574,13 @@ func TestRunnerRunJSONSchemaRepairCarriesStructuredOutputSpec(t *testing.T) {
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: `{"events":[1]}`}}
runner := NewRunnerWithRepairer(
&fakePromptRepo{def: def},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
defaultArtifactReader(),
defaultRenderer(),
llmClient,
validator,
repairer,
)
repairer)
_, err := runner.Run(context.Background(), domain.RunRequest{
PromptID: "p",
@@ -1629,13 +1657,12 @@ func TestResolveExecutionTargetProfileValuesPopulateAllSupportedFields(t *testin
ServiceTier: "priority",
ReasoningEffort: "low",
APIKeyEnv: "PROFILE_KEY",
APIKeyRequired: true,
ExtraParams: map[string]any{
"profile_option": "enabled",
},
}
target, presence, err := resolveExecutionTarget(profileValue, nil)
target, presence, err := resolveExecutionTarget(nil, profileValue, nil)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
@@ -1690,7 +1717,7 @@ func TestResolveExecutionTargetRuntimeOverridesBeatProfileForAllOverrideableFiel
},
}
target, presence, err := resolveExecutionTarget(profileValue, override)
target, presence, err := resolveExecutionTarget(nil, profileValue, override)
if err != nil {
t.Fatalf("expected no error, got %v", err)
}
@@ -1813,6 +1840,118 @@ func defaultExecutionProfile() *domain.ExecutionProfile {
}
}
func TestResolveExecutionTargetUsesBackendProfileAndRequestPrecedence(t *testing.T) {
backendValue := &domain.Backend{
ID: "custom",
Endpoint: "http://backend/v1",
APIKeyEnv: "BACKEND_KEY",
ExtraParams: map[string]any{"backend": true},
}
profileValue := &domain.ExecutionProfile{
ID: "exec",
BackendID: "custom",
Endpoint: "http://profile/v1",
Model: "profile-model",
APIKeyEnv: "PROFILE_KEY",
ExtraParams: map[string]any{"profile": true},
}
override := &domain.ExecutionTargetOverride{
Endpoint: "http://request/v1",
APIKeyEnv: "REQUEST_KEY",
ExtraParams: map[string]any{"request": true},
}
target, _, err := resolveExecutionTarget(backendValue, profileValue, override)
if err != nil {
t.Fatalf("resolve target: %v", err)
}
if target.BackendID != "custom" {
t.Fatalf("endpoint override changed backend identity: %+v", target)
}
if target.Endpoint != "http://request/v1" || target.APIKeyEnv != "REQUEST_KEY" {
t.Fatalf("request values did not win: %+v", target)
}
if !reflect.DeepEqual(target.ExtraParams, map[string]any{"request": true}) {
t.Fatalf("expected whole-map request replacement, got %#v", target.ExtraParams)
}
target, _, err = resolveExecutionTarget(backendValue, &domain.ExecutionProfile{
ID: "exec", BackendID: "custom", Model: "profile-model",
}, nil)
if err != nil {
t.Fatalf("resolve backend defaults: %v", err)
}
if target.Endpoint != backendValue.Endpoint ||
target.APIKeyEnv != backendValue.APIKeyEnv ||
!reflect.DeepEqual(target.ExtraParams, backendValue.ExtraParams) {
t.Fatalf("backend defaults were not inherited: %+v", target)
}
}
func TestRunnerPrepareBackendResolutionAndCredentialPrecedence(t *testing.T) {
resolver := fakeBackendResolver{backends: map[string]domain.Backend{
"custom": {ID: "custom", Endpoint: "http://backend/v1", APIKeyEnv: "BACKEND_KEY"},
}}
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
t.Run("unknown backend is a profile load failure", func(t *testing.T) {
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", BackendID: "unknown", Model: "model"},
}}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if !errors.Is(err, ErrProfileLoad) {
t.Fatalf("expected ErrProfileLoad, got %v", err)
}
})
t.Run("nil resolver is a profile load failure", func(t *testing.T) {
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", BackendID: "custom", Model: "model"},
}}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if !errors.Is(err, ErrProfileLoad) {
t.Fatalf("expected ErrProfileLoad, got %v", err)
}
})
t.Run("request environment wins", func(t *testing.T) {
t.Setenv("REQUEST_KEY", "request-secret")
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", BackendID: "custom", Model: "model", APIKeyEnv: "PROFILE_KEY"},
}}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(),
Execution: &domain.ExecutionTargetOverride{APIKeyEnv: "REQUEST_KEY"},
})
if err != nil {
t.Fatalf("prepare: %v", err)
}
if prepared.EffectiveModelParams.APIKeyEnv != "REQUEST_KEY" {
t.Fatalf("unexpected credential source: %+v", prepared.EffectiveModelParams)
}
})
t.Run("required direct key clears inherited environment", func(t *testing.T) {
t.Setenv("BACKEND_KEY", "backend-secret")
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
"exec": {ID: "exec", BackendID: "custom", Model: "model", APIKeyRequired: true},
}}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
if !errors.Is(err, ErrAPIKeyRequired) {
t.Fatalf("expected ErrAPIKeyRequired, got %v", err)
}
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(), APIKey: "direct-secret",
})
if err != nil {
t.Fatalf("prepare with direct key: %v", err)
}
if prepared.EffectiveModelParams.APIKeyEnv != "" {
t.Fatalf("expected inherited environment name to be cleared, got %q", prepared.EffectiveModelParams.APIKeyEnv)
}
})
}
func defaultArtifactReader() *fakeArtifactReader {
return &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{
"a://ok": {Body: []byte("x"), Hash: hashString("x")},
@@ -1838,10 +1977,11 @@ func intPtr(v int) *int {
func newMinimalRunner(promptRepo *fakePromptRepo, execRepo *fakeExecutionProfileRepo) *Runner {
return NewRunner(
promptRepo,
execRepo,
execRepo, nil,
defaultArtifactReader(),
defaultRenderer(),
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
nil,
)
nil)
}