Resolve profiles through built-in backends
This commit is contained in:
@@ -81,6 +81,7 @@ type RunResult struct {
|
||||
PromptHash string
|
||||
RenderedPromptHash string
|
||||
SelectedProfileID string
|
||||
SelectedBackendID string
|
||||
ModelName string
|
||||
Endpoint string
|
||||
EffectiveModelParams ExecutionTarget
|
||||
@@ -98,6 +99,7 @@ type PreparedRun struct {
|
||||
PromptVersion string `json:"prompt_version,omitempty"`
|
||||
PromptHash string `json:"prompt_hash,omitempty"`
|
||||
SelectedProfileID string `json:"selected_profile_id"`
|
||||
SelectedBackendID string `json:"selected_backend_id,omitempty"`
|
||||
EffectiveModelParams ExecutionTarget `json:"effective_model_params"`
|
||||
TargetPresence ExecutionTargetPresence `json:"-"`
|
||||
OutputContract OutputContract `json:"output_contract"`
|
||||
@@ -168,6 +170,7 @@ type Backend struct {
|
||||
// ExecutionProfile describes how and where to execute a model.
|
||||
type ExecutionProfile struct {
|
||||
ID string `yaml:"id"`
|
||||
BackendID string `yaml:"backend"`
|
||||
Endpoint string `yaml:"endpoint"`
|
||||
Model string `yaml:"model"`
|
||||
Temperature float64 `yaml:"temperature"`
|
||||
@@ -206,6 +209,7 @@ type ExecutionTargetPresence struct {
|
||||
|
||||
// ExecutionTarget represents effective model runtime settings for a run.
|
||||
type ExecutionTarget struct {
|
||||
BackendID string `yaml:"backend" json:"backend_id,omitempty"`
|
||||
Endpoint string `yaml:"endpoint" json:"endpoint"`
|
||||
Model string `yaml:"model" json:"model"`
|
||||
Temperature float64 `yaml:"temperature" json:"temperature"`
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
id: aion-2
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: aion-labs/aion-2.0
|
||||
temperature: 0.72
|
||||
reasoning_effort: high
|
||||
top_p: 0.95
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: claude-fable-latest
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "~anthropic/claude-fable-latest"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 600
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: claude-haiku-latest
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "~anthropic/claude-haiku-latest"
|
||||
reasoning_effort: medium
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: claude-opus-latest
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "~anthropic/claude-opus-latest"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: claude-sonnet-latest
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "~anthropic/claude-sonnet-latest"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: deepseek-3-2
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: deepseek/deepseek-v3.2
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: deepseek-4-flash
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: deepseek/deepseek-v4-flash
|
||||
#reasoning_effort: medium
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: deepseek-4-pro
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: deepseek/deepseek-v4-pro
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
id: gemini-2-flash-lite
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "google/gemini-2.5-flash-lite"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
id: gemini-2-flash
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "google/gemini-2.5-flash"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
id: gemini-2-pro
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "google/gemini-2.5-pro"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
id: gemini-3-flash-lite
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "google/gemini-3.1-flash-lite"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
id: gemini-flash-latest
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "~google/gemini-flash-latest"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
id: gemini-pro-latest
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "~google/gemini-pro-latest"
|
||||
#temperature: 0.15
|
||||
reasoning_effort: high
|
||||
#top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
id: gemma-4-31b
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: google/gemma-4-31b-it:exacto
|
||||
temperature: 0.15
|
||||
reasoning_effort: high
|
||||
top_p: 0.98
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
id: minimax-m2
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: minimax/minimax-m2.5
|
||||
temperature: 0.5
|
||||
reasoning_effort: high
|
||||
top_p: 0.95
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
id: minimax-m3
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: minimax/minimax-m3
|
||||
#temperature: 0.5
|
||||
reasoning_effort: high
|
||||
#top_p: 0.95
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: mistral-large-2512
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: mistralai/mistral-large-2512
|
||||
temperature: 0.15
|
||||
top_p: 0.98
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
id: mistral-medium-3-5
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: mistralai/mistral-medium-3-5
|
||||
temperature: 0.15
|
||||
reasoning_effort: high
|
||||
top_p: 0.98
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: mistral-small-3
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: mistralai/mistral-small-3.2-24b-instruct
|
||||
temperature: 0.05
|
||||
top_p: 1.0
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
|
||||
@@ -1,8 +1,7 @@
|
||||
id: mistral-small-4
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: mistralai/mistral-small-2603
|
||||
temperature: 0.1
|
||||
reasoning_effort: high
|
||||
top_p: 0.98
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: nemotron-3-ultra
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: nvidia/nemotron-3-ultra-550b-a55b
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 180
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: gpt-5-mini
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "openai/gpt-5.4-mini"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
id: gpt-5-nano
|
||||
endpoint: https://openrouter.ai/api/v1
|
||||
backend: openrouter
|
||||
model: "openai/gpt-5.4-nano"
|
||||
reasoning_effort: high
|
||||
timeout_seconds: 240
|
||||
api_key_env: OPENROUTER_API_KEY
|
||||
service_tier: flex
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/backend"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/domain"
|
||||
"gitea.maximumdirect.net/eric/promptkit/internal/profile"
|
||||
"gopkg.in/yaml.v3"
|
||||
@@ -28,6 +29,12 @@ func TestBuiltInProfilesValidateThroughRepository(t *testing.T) {
|
||||
if p.ID != id {
|
||||
t.Fatalf("expected profile id %q, got %q", id, p.ID)
|
||||
}
|
||||
if p.BackendID != backend.OpenRouterID {
|
||||
t.Fatalf("expected profile %q to select %q, got %q", id, backend.OpenRouterID, p.BackendID)
|
||||
}
|
||||
if p.Endpoint != "" || p.APIKeyEnv != "" {
|
||||
t.Fatalf("expected profile %q to inherit backend connection settings, got endpoint=%q api_key_env=%q", id, p.Endpoint, p.APIKeyEnv)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -60,6 +67,15 @@ func loadBuiltInProfileIDs(t *testing.T) map[string]string {
|
||||
if _, ok := raw["api_key"]; ok {
|
||||
t.Fatalf("built-in profile %s contains raw api_key", name)
|
||||
}
|
||||
if raw["backend"] != backend.OpenRouterID {
|
||||
t.Fatalf("built-in profile %s does not select %q", name, backend.OpenRouterID)
|
||||
}
|
||||
if _, ok := raw["endpoint"]; ok {
|
||||
t.Fatalf("built-in profile %s repeats endpoint", name)
|
||||
}
|
||||
if _, ok := raw["api_key_env"]; ok {
|
||||
t.Fatalf("built-in profile %s repeats api_key_env", name)
|
||||
}
|
||||
id, ok := raw["id"].(string)
|
||||
if !ok || strings.TrimSpace(id) == "" {
|
||||
t.Fatalf("built-in profile %s has missing id", name)
|
||||
|
||||
@@ -121,6 +121,7 @@ func loadProfile(ctx context.Context, fsys fs.FS, root string, id string) (*doma
|
||||
if prof.ID != id {
|
||||
continue
|
||||
}
|
||||
prof.BackendID = strings.TrimSpace(prof.BackendID)
|
||||
if err := validateProfile(&prof); err != nil {
|
||||
if errors.Is(err, ErrRawAPIKeyNotAllowed) {
|
||||
return nil, fmt.Errorf("%w: %s", err, relPath)
|
||||
@@ -189,8 +190,8 @@ func validateProfile(p *domain.ExecutionProfile) error {
|
||||
if strings.TrimSpace(p.ID) == "" {
|
||||
return errors.New("id is required")
|
||||
}
|
||||
if strings.TrimSpace(p.Endpoint) == "" {
|
||||
return errors.New("endpoint is required")
|
||||
if strings.TrimSpace(p.BackendID) == "" && strings.TrimSpace(p.Endpoint) == "" {
|
||||
return errors.New("backend or endpoint is required")
|
||||
}
|
||||
if strings.TrimSpace(p.Model) == "" {
|
||||
return errors.New("model is required")
|
||||
|
||||
@@ -52,6 +52,43 @@ func TestFilesystemRepository_GetProfile(t *testing.T) {
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("backend and endpoint connection matrix", func(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
connection string
|
||||
wantBackend string
|
||||
wantEndpoint string
|
||||
wantErr bool
|
||||
}{
|
||||
{name: "backend only", connection: "backend: ' openrouter '", wantBackend: "openrouter"},
|
||||
{name: "endpoint only", connection: "endpoint: http://localhost:8000/v1", wantEndpoint: "http://localhost:8000/v1"},
|
||||
{name: "both", connection: "backend: openrouter\nendpoint: http://localhost:8000/v1", wantBackend: "openrouter", wantEndpoint: "http://localhost:8000/v1"},
|
||||
{name: "neither", wantErr: true},
|
||||
{name: "blank backend", connection: "backend: ' '", wantErr: true},
|
||||
}
|
||||
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
id := "connection-" + strings.ReplaceAll(tt.name, " ", "-")
|
||||
writeProfileTestFile(t, filepath.Join(tmpDir, id+".yaml"), "id: "+id+"\nmodel: model\n"+tt.connection+"\n")
|
||||
|
||||
p, err := repo.GetProfile(ctx, id)
|
||||
if tt.wantErr {
|
||||
if !errors.Is(err, ErrInvalidProfile) {
|
||||
t.Fatalf("expected ErrInvalidProfile, got %v", err)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatalf("expected profile to load, got %v", err)
|
||||
}
|
||||
if p.BackendID != tt.wantBackend || p.Endpoint != tt.wantEndpoint {
|
||||
t.Fatalf("unexpected connection values: backend=%q endpoint=%q", p.BackendID, p.Endpoint)
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("valid profile with api_key_env", func(t *testing.T) {
|
||||
p, err := repo.GetProfile(ctx, "local-secure")
|
||||
if err != nil {
|
||||
|
||||
@@ -40,6 +40,7 @@ var (
|
||||
type Runner struct {
|
||||
promptDefs promptdef.Repository
|
||||
profiles profile.Repository
|
||||
backends BackendResolver
|
||||
artifacts artifact.Reader
|
||||
renderer prompt.Renderer
|
||||
llm llm.Client
|
||||
@@ -47,20 +48,27 @@ type Runner struct {
|
||||
repairer OutputRepairer
|
||||
}
|
||||
|
||||
// BackendResolver resolves one normalized backend ID.
|
||||
type BackendResolver interface {
|
||||
GetBackend(string) (domain.Backend, error)
|
||||
}
|
||||
|
||||
func NewRunner(
|
||||
promptDefs promptdef.Repository,
|
||||
profiles profile.Repository,
|
||||
backends BackendResolver,
|
||||
artifacts artifact.Reader,
|
||||
renderer prompt.Renderer,
|
||||
llmClient llm.Client,
|
||||
validator validate.Validator,
|
||||
) *Runner {
|
||||
return NewRunnerWithRepairer(promptDefs, profiles, artifacts, renderer, llmClient, validator, nil)
|
||||
return NewRunnerWithRepairer(promptDefs, profiles, backends, artifacts, renderer, llmClient, validator, nil)
|
||||
}
|
||||
|
||||
func NewRunnerWithRepairer(
|
||||
promptDefs promptdef.Repository,
|
||||
profiles profile.Repository,
|
||||
backends BackendResolver,
|
||||
artifacts artifact.Reader,
|
||||
renderer prompt.Renderer,
|
||||
llmClient llm.Client,
|
||||
@@ -70,6 +78,7 @@ func NewRunnerWithRepairer(
|
||||
return &Runner{
|
||||
promptDefs: promptDefs,
|
||||
profiles: profiles,
|
||||
backends: backends,
|
||||
artifacts: artifacts,
|
||||
renderer: renderer,
|
||||
llm: llmClient,
|
||||
@@ -153,6 +162,7 @@ func (r *Runner) Run(ctx context.Context, req domain.RunRequest) (*domain.RunRes
|
||||
PromptHash: prepared.PromptHash,
|
||||
RenderedPromptHash: prepared.RenderedPromptHash,
|
||||
SelectedProfileID: prepared.SelectedProfileID,
|
||||
SelectedBackendID: prepared.SelectedBackendID,
|
||||
ModelName: prepared.EffectiveModelParams.Model,
|
||||
Endpoint: prepared.EffectiveModelParams.Endpoint,
|
||||
EffectiveModelParams: prepared.EffectiveModelParams,
|
||||
@@ -193,7 +203,20 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
|
||||
return nil, fmt.Errorf("%w: %w", ErrProfileLoad, err)
|
||||
}
|
||||
|
||||
effectiveModel, targetPresence, err := resolveExecutionTarget(execProfile, req.Execution)
|
||||
var selectedBackend *domain.Backend
|
||||
if backendID := strings.TrimSpace(execProfile.BackendID); backendID != "" {
|
||||
execProfile.BackendID = backendID
|
||||
if r.backends == nil {
|
||||
return nil, fmt.Errorf("%w: backend %q cannot be resolved", ErrProfileLoad, backendID)
|
||||
}
|
||||
resolvedBackend, resolveErr := r.backends.GetBackend(backendID)
|
||||
if resolveErr != nil {
|
||||
return nil, fmt.Errorf("%w: backend %q: %w", ErrProfileLoad, backendID, resolveErr)
|
||||
}
|
||||
selectedBackend = &resolvedBackend
|
||||
}
|
||||
|
||||
effectiveModel, targetPresence, err := resolveExecutionTarget(selectedBackend, execProfile, req.Execution)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err)
|
||||
}
|
||||
@@ -239,6 +262,7 @@ func (r *Runner) Prepare(ctx context.Context, req domain.RunRequest) (*domain.Pr
|
||||
PromptVersion: def.Version,
|
||||
PromptHash: promptDefinitionHash,
|
||||
SelectedProfileID: selectedProfileID,
|
||||
SelectedBackendID: effectiveModel.BackendID,
|
||||
EffectiveModelParams: effectiveModel,
|
||||
TargetPresence: targetPresence,
|
||||
OutputContract: effectiveContract,
|
||||
@@ -338,6 +362,9 @@ func (r *Runner) shouldAttemptRepair(contract domain.OutputContract, validationR
|
||||
|
||||
func mergeExecutionTarget(base domain.ExecutionTarget, override domain.ExecutionTarget) domain.ExecutionTarget {
|
||||
out := base
|
||||
if strings.TrimSpace(override.BackendID) != "" {
|
||||
out.BackendID = override.BackendID
|
||||
}
|
||||
if override.Endpoint != "" {
|
||||
out.Endpoint = override.Endpoint
|
||||
}
|
||||
@@ -367,6 +394,7 @@ func mergeExecutionTarget(base domain.ExecutionTarget, override domain.Execution
|
||||
}
|
||||
if override.APIKeyRequired {
|
||||
out.APIKeyRequired = true
|
||||
out.APIKeyEnv = ""
|
||||
}
|
||||
if len(override.ExtraParams) > 0 {
|
||||
out.ExtraParams = copyExtraParams(override.ExtraParams)
|
||||
@@ -426,8 +454,9 @@ func mergeExecutionTargetOverride(base domain.ExecutionTarget, override domain.E
|
||||
return out, presence, nil
|
||||
}
|
||||
|
||||
func resolveExecutionTarget(profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) {
|
||||
func resolveExecutionTarget(backendValue *domain.Backend, profileValue *domain.ExecutionProfile, override *domain.ExecutionTargetOverride) (domain.ExecutionTarget, domain.ExecutionTargetPresence, error) {
|
||||
out := defaults.ExecutionTargetDefault()
|
||||
out = mergeExecutionTarget(out, backendToTarget(backendValue))
|
||||
out = mergeExecutionTarget(out, executionProfileToTarget(profileValue))
|
||||
var presence domain.ExecutionTargetPresence
|
||||
if override != nil {
|
||||
@@ -461,8 +490,13 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
|
||||
if p == nil {
|
||||
return domain.ExecutionTarget{}
|
||||
}
|
||||
endpoint := p.Endpoint
|
||||
if strings.TrimSpace(endpoint) == "" {
|
||||
endpoint = ""
|
||||
}
|
||||
return domain.ExecutionTarget{
|
||||
Endpoint: p.Endpoint,
|
||||
BackendID: p.BackendID,
|
||||
Endpoint: endpoint,
|
||||
Model: p.Model,
|
||||
Temperature: p.Temperature,
|
||||
MaxTokens: p.MaxTokens,
|
||||
@@ -476,6 +510,18 @@ func executionProfileToTarget(p *domain.ExecutionProfile) domain.ExecutionTarget
|
||||
}
|
||||
}
|
||||
|
||||
func backendToTarget(value *domain.Backend) domain.ExecutionTarget {
|
||||
if value == nil {
|
||||
return domain.ExecutionTarget{}
|
||||
}
|
||||
return domain.ExecutionTarget{
|
||||
BackendID: value.ID,
|
||||
Endpoint: value.Endpoint,
|
||||
APIKeyEnv: value.APIKeyEnv,
|
||||
ExtraParams: copyExtraParams(value.ExtraParams),
|
||||
}
|
||||
}
|
||||
|
||||
func copyExtraParams(src map[string]any) map[string]any {
|
||||
if len(src) == 0 {
|
||||
return nil
|
||||
|
||||
@@ -34,6 +34,18 @@ type fakeExecutionProfileRepo struct {
|
||||
lastID string
|
||||
}
|
||||
|
||||
type fakeBackendResolver struct {
|
||||
backends map[string]domain.Backend
|
||||
}
|
||||
|
||||
func (f fakeBackendResolver) GetBackend(id string) (domain.Backend, error) {
|
||||
value, ok := f.backends[id]
|
||||
if !ok {
|
||||
return domain.Backend{}, errors.New("backend not found")
|
||||
}
|
||||
return value, nil
|
||||
}
|
||||
|
||||
func (f *fakeExecutionProfileRepo) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) {
|
||||
f.lastID = id
|
||||
if f.err != nil {
|
||||
@@ -167,7 +179,7 @@ func TestRunnerPrepareWithExplicitProfileSelection(t *testing.T) {
|
||||
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
||||
llmClient := &fakeLLM{forbid: true}
|
||||
|
||||
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil)
|
||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
PromptVersion: "1",
|
||||
@@ -257,7 +269,7 @@ func TestRunnerPrepareSelectedProfileDoesNotExistFails(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunnerPreparePromptLoadFailure(t *testing.T) {
|
||||
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
|
||||
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
|
||||
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p"})
|
||||
if !errors.Is(err, ErrPromptLoad) {
|
||||
t.Fatalf("expected ErrPromptLoad, got %v", err)
|
||||
@@ -281,7 +293,7 @@ func TestRunnerPrepareRuntimeOverrideBeatsSelectedProfileValue(t *testing.T) {
|
||||
ServiceTier: "priority",
|
||||
},
|
||||
}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||
|
||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -379,12 +391,12 @@ func TestRunnerPrepareRequestNumericOverridePresence(t *testing.T) {
|
||||
TopP: 0.8,
|
||||
TimeoutSeconds: 45,
|
||||
},
|
||||
}},
|
||||
}}, nil,
|
||||
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
&fakeLLM{forbid: true},
|
||||
nil,
|
||||
)
|
||||
nil)
|
||||
|
||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -426,12 +438,12 @@ func TestRunnerPrepareInvalidRequestNumericOverridesFail(t *testing.T) {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
&fakeLLM{forbid: true},
|
||||
nil,
|
||||
)
|
||||
nil)
|
||||
|
||||
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -458,7 +470,7 @@ func TestRunnerPrepareSelectedProfileBeatsBuiltInDefault(t *testing.T) {
|
||||
ServiceTier: "priority",
|
||||
},
|
||||
}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||
|
||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -495,12 +507,12 @@ func TestRunnerPrepareFileBackedPromptBodiesRenderCorrectly(t *testing.T) {
|
||||
llmClient := &fakeLLM{forbid: true}
|
||||
runner := NewRunner(
|
||||
promptdef.NewFilesystemRepository(promptDir),
|
||||
profile.NewFilesystemRepository(profileDir),
|
||||
profile.NewFilesystemRepository(profileDir), nil,
|
||||
|
||||
reader,
|
||||
prompt.NewGoRenderer(),
|
||||
llmClient,
|
||||
nil,
|
||||
)
|
||||
nil)
|
||||
|
||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||
PromptID: "valid-file-backed",
|
||||
@@ -526,12 +538,13 @@ func TestRunnerPrepareRequiredInputMissingFails(t *testing.T) {
|
||||
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: def},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||
|
||||
defaultArtifactReader(),
|
||||
prompt.NewGoRenderer(),
|
||||
&fakeLLM{forbid: true},
|
||||
nil,
|
||||
)
|
||||
nil)
|
||||
|
||||
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
@@ -552,12 +565,13 @@ func TestRunnerPrepareUnknownTemplateInputReferenceFails(t *testing.T) {
|
||||
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: def},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||
|
||||
defaultArtifactReader(),
|
||||
prompt.NewGoRenderer(),
|
||||
&fakeLLM{forbid: true},
|
||||
nil,
|
||||
)
|
||||
nil)
|
||||
|
||||
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
@@ -579,7 +593,7 @@ func TestRunnerPrepareAPIKeyEnvNameIncludedButNotResolvedValue(t *testing.T) {
|
||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName},
|
||||
}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||
|
||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||
if err != nil {
|
||||
@@ -607,12 +621,12 @@ func TestRunnerPrepareJSONSchemaBuildsStructuredOutputSpec(t *testing.T) {
|
||||
}
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: def},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
&fakeLLM{forbid: true},
|
||||
validator,
|
||||
)
|
||||
validator)
|
||||
|
||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -651,12 +665,12 @@ func TestRunnerPrepareJSONSchemaSchemaLoadFailureReturnsValidationError(t *testi
|
||||
validator := &fakeValidator{schemaErr: errors.New("schema unavailable")}
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: def},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
&fakeLLM{forbid: true},
|
||||
validator,
|
||||
)
|
||||
validator)
|
||||
|
||||
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -681,12 +695,12 @@ func TestRunnerRunJSONSchemaSchemaLoadFailureFailsBeforeLLM(t *testing.T) {
|
||||
validator := &fakeValidator{schemaErr: errors.New("schema unavailable")}
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: def},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
llmClient,
|
||||
validator,
|
||||
)
|
||||
validator)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -836,7 +850,7 @@ func TestRunnerRunSuccessful(t *testing.T) {
|
||||
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{SessionID: "session-123", Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap", Usage: domain.TokenUsage{TotalTokens: 7}}}
|
||||
|
||||
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil)
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
PromptVersion: "1",
|
||||
@@ -915,7 +929,7 @@ func TestRunnerRunPassesExtraParamsToGenerateRequestTarget(t *testing.T) {
|
||||
},
|
||||
}}
|
||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -941,7 +955,7 @@ func TestRunnerRunAndPrepareResolveSameProfileAndEffectiveSettings(t *testing.T)
|
||||
}}
|
||||
renderer := &fakeRenderer{rendered: &domain.RenderedPrompt{Messages: []domain.RenderedMessage{{Role: "system", Content: "sys"}, {Role: "user", Content: "usr"}}}}
|
||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "# recap"}}
|
||||
runner := NewRunner(promptRepo, execRepo, reader, renderer, llmClient, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, reader, renderer, llmClient, nil)
|
||||
|
||||
req := domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -1071,7 +1085,7 @@ func TestRunnerRunExplicitRuntimeOverrideBeatsSelectedProfileValue(t *testing.T)
|
||||
},
|
||||
}}
|
||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -1115,7 +1129,7 @@ func TestRunnerRunSelectedProfileBeatsBuiltInDefault(t *testing.T) {
|
||||
},
|
||||
}}
|
||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -1142,7 +1156,7 @@ func TestRunnerRunBuiltInDefaultsUsedWhenProfileOmitsOptionalFields(t *testing.T
|
||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"},
|
||||
}}
|
||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -1172,7 +1186,7 @@ func TestRunnerRunAPIKeyEnvResolvesFromEnvironment(t *testing.T) {
|
||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_TEST_API_KEY"},
|
||||
}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||
if err != nil {
|
||||
@@ -1188,7 +1202,7 @@ func TestRunnerRunAPIKeyEnvMissingEnvironmentValueFailsClearly(t *testing.T) {
|
||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_MISSING_KEY"},
|
||||
}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||
if !errors.Is(err, ErrInvalidRequest) {
|
||||
@@ -1209,7 +1223,7 @@ func TestRunnerRunDirectAPIKeyBypassesMissingEnvAndReachesLLM(t *testing.T) {
|
||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: "PROMPTKIT_MISSING_KEY"},
|
||||
}}
|
||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -1233,7 +1247,7 @@ func TestRunnerPrepareAPIKeyRequiredFailsWithoutDirectKey(t *testing.T) {
|
||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true},
|
||||
}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||
|
||||
_, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -1252,7 +1266,7 @@ func TestRunnerRunAPIKeyRequiredSucceedsWithDirectKey(t *testing.T) {
|
||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyRequired: true},
|
||||
}}
|
||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), llmClient, nil)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -1279,7 +1293,7 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideWorks(t *testing.T) {
|
||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model"},
|
||||
}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -1304,7 +1318,7 @@ func TestRunnerRunRuntimeAPIKeyEnvOverrideBeatsProfile(t *testing.T) {
|
||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: profileEnv},
|
||||
}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -1328,7 +1342,7 @@ func TestRunnerRunAPIKeyValueNotPresentInMetadata(t *testing.T) {
|
||||
execRepo := &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", APIKeyEnv: envName},
|
||||
}}
|
||||
runner := NewRunner(promptRepo, execRepo, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||
runner := NewRunner(promptRepo, execRepo, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}}, nil)
|
||||
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||
if err != nil {
|
||||
@@ -1344,7 +1358,7 @@ func TestRunnerRunAPIKeyValueNotPresentInMetadata(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestRunnerRunPromptLoadFailure(t *testing.T) {
|
||||
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
|
||||
runner := NewRunner(&fakePromptRepo{err: errors.New("boom")}, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{}, nil)
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{PromptID: "p"})
|
||||
if !errors.Is(err, ErrPromptLoad) {
|
||||
t.Fatalf("expected ErrPromptLoad, got %v", err)
|
||||
@@ -1357,12 +1371,12 @@ func TestRunnerRunPromptLoadFailure(t *testing.T) {
|
||||
func TestRunnerRunArtifactLoadFailure(t *testing.T) {
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||
|
||||
&fakeArtifactReader{errByURI: map[string]error{"a://bad": errors.New("read failed")}},
|
||||
&fakeRenderer{rendered: &domain.RenderedPrompt{}},
|
||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
|
||||
nil,
|
||||
)
|
||||
nil)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -1377,12 +1391,13 @@ func TestRunnerRunArtifactLoadFailure(t *testing.T) {
|
||||
func TestRunnerRunPromptRenderFailure(t *testing.T) {
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||
|
||||
defaultArtifactReader(),
|
||||
&fakeRenderer{err: errors.New("render failed")},
|
||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
|
||||
nil,
|
||||
)
|
||||
nil)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
@@ -1396,12 +1411,13 @@ func TestRunnerRunPromptRenderFailure(t *testing.T) {
|
||||
func TestRunnerRunLLMFailure(t *testing.T) {
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
&fakeLLM{err: errors.New("llm failed")},
|
||||
nil,
|
||||
)
|
||||
nil)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
@@ -1415,12 +1431,13 @@ func TestRunnerRunLLMFailure(t *testing.T) {
|
||||
func TestRunnerRunCancellationPreservesGenerationCategory(t *testing.T) {
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "ignored"}},
|
||||
nil,
|
||||
)
|
||||
nil)
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
|
||||
@@ -1440,12 +1457,13 @@ func TestRunnerRunCancellationPreservesGenerationCategory(t *testing.T) {
|
||||
func TestRunnerRunLLMInvalidRequestMapsToUsecaseInvalidRequest(t *testing.T) {
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
&fakeLLM{err: llm.ErrInvalidRequest},
|
||||
nil,
|
||||
)
|
||||
nil)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
@@ -1463,12 +1481,13 @@ func TestRunnerRunValidationStillWorks(t *testing.T) {
|
||||
validator := &fakeValidator{result: domain.ValidationResult{Status: domain.ValidationFailed, Mode: domain.ValidationBasic, Errors: []string{"bad"}, IsValid: false}}
|
||||
runner := NewRunner(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationBasic, 0)},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "raw output"}},
|
||||
validator,
|
||||
)
|
||||
validator)
|
||||
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
@@ -1489,14 +1508,17 @@ func TestRunnerRunStructuredRepairRemainsBoundedAndUsesEffectiveModelSettings(t
|
||||
runner := NewRunnerWithRepairer(
|
||||
&fakePromptRepo{def: promptDef(domain.FormatJSON, domain.ValidationJSON, 1)},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||
"exec": {ID: "exec", Endpoint: "http://profile/v1", Model: "profile-model", TimeoutSeconds: 55},
|
||||
"exec": {ID: "exec", BackendID: "custom", Model: "profile-model", TimeoutSeconds: 55},
|
||||
}}, fakeBackendResolver{backends: map[string]domain.Backend{
|
||||
"custom": {ID: "custom", Endpoint: "http://backend/v1"},
|
||||
}},
|
||||
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
llmClient,
|
||||
validate.NewStandardValidator("."),
|
||||
repairer,
|
||||
)
|
||||
repairer)
|
||||
|
||||
res, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
ProfileID: "exec",
|
||||
@@ -1518,6 +1540,12 @@ func TestRunnerRunStructuredRepairRemainsBoundedAndUsesEffectiveModelSettings(t
|
||||
if repairer.reqs[0].Target.TimeoutSeconds != 22 {
|
||||
t.Fatalf("expected repair to use effective timeout, got %d", repairer.reqs[0].Target.TimeoutSeconds)
|
||||
}
|
||||
if llmClient.lastReq.Target.BackendID != "custom" ||
|
||||
repairer.reqs[0].Target.BackendID != "custom" ||
|
||||
res.SelectedBackendID != "custom" {
|
||||
t.Fatalf("expected backend identity in generation, repair, and result: generate=%q repair=%q result=%q",
|
||||
llmClient.lastReq.Target.BackendID, repairer.reqs[0].Target.BackendID, res.SelectedBackendID)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerRunJSONSchemaRepairCarriesStructuredOutputSpec(t *testing.T) {
|
||||
@@ -1546,13 +1574,13 @@ func TestRunnerRunJSONSchemaRepairCarriesStructuredOutputSpec(t *testing.T) {
|
||||
llmClient := &fakeLLM{resp: &domain.GenerateResponse{Content: `{"events":[1]}`}}
|
||||
runner := NewRunnerWithRepairer(
|
||||
&fakePromptRepo{def: def},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}},
|
||||
&fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{"exec": defaultExecutionProfile()}}, nil,
|
||||
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
llmClient,
|
||||
validator,
|
||||
repairer,
|
||||
)
|
||||
repairer)
|
||||
|
||||
_, err := runner.Run(context.Background(), domain.RunRequest{
|
||||
PromptID: "p",
|
||||
@@ -1629,13 +1657,12 @@ func TestResolveExecutionTargetProfileValuesPopulateAllSupportedFields(t *testin
|
||||
ServiceTier: "priority",
|
||||
ReasoningEffort: "low",
|
||||
APIKeyEnv: "PROFILE_KEY",
|
||||
APIKeyRequired: true,
|
||||
ExtraParams: map[string]any{
|
||||
"profile_option": "enabled",
|
||||
},
|
||||
}
|
||||
|
||||
target, presence, err := resolveExecutionTarget(profileValue, nil)
|
||||
target, presence, err := resolveExecutionTarget(nil, profileValue, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
@@ -1690,7 +1717,7 @@ func TestResolveExecutionTargetRuntimeOverridesBeatProfileForAllOverrideableFiel
|
||||
},
|
||||
}
|
||||
|
||||
target, presence, err := resolveExecutionTarget(profileValue, override)
|
||||
target, presence, err := resolveExecutionTarget(nil, profileValue, override)
|
||||
if err != nil {
|
||||
t.Fatalf("expected no error, got %v", err)
|
||||
}
|
||||
@@ -1813,6 +1840,118 @@ func defaultExecutionProfile() *domain.ExecutionProfile {
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolveExecutionTargetUsesBackendProfileAndRequestPrecedence(t *testing.T) {
|
||||
backendValue := &domain.Backend{
|
||||
ID: "custom",
|
||||
Endpoint: "http://backend/v1",
|
||||
APIKeyEnv: "BACKEND_KEY",
|
||||
ExtraParams: map[string]any{"backend": true},
|
||||
}
|
||||
profileValue := &domain.ExecutionProfile{
|
||||
ID: "exec",
|
||||
BackendID: "custom",
|
||||
Endpoint: "http://profile/v1",
|
||||
Model: "profile-model",
|
||||
APIKeyEnv: "PROFILE_KEY",
|
||||
ExtraParams: map[string]any{"profile": true},
|
||||
}
|
||||
override := &domain.ExecutionTargetOverride{
|
||||
Endpoint: "http://request/v1",
|
||||
APIKeyEnv: "REQUEST_KEY",
|
||||
ExtraParams: map[string]any{"request": true},
|
||||
}
|
||||
|
||||
target, _, err := resolveExecutionTarget(backendValue, profileValue, override)
|
||||
if err != nil {
|
||||
t.Fatalf("resolve target: %v", err)
|
||||
}
|
||||
if target.BackendID != "custom" {
|
||||
t.Fatalf("endpoint override changed backend identity: %+v", target)
|
||||
}
|
||||
if target.Endpoint != "http://request/v1" || target.APIKeyEnv != "REQUEST_KEY" {
|
||||
t.Fatalf("request values did not win: %+v", target)
|
||||
}
|
||||
if !reflect.DeepEqual(target.ExtraParams, map[string]any{"request": true}) {
|
||||
t.Fatalf("expected whole-map request replacement, got %#v", target.ExtraParams)
|
||||
}
|
||||
|
||||
target, _, err = resolveExecutionTarget(backendValue, &domain.ExecutionProfile{
|
||||
ID: "exec", BackendID: "custom", Model: "profile-model",
|
||||
}, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("resolve backend defaults: %v", err)
|
||||
}
|
||||
if target.Endpoint != backendValue.Endpoint ||
|
||||
target.APIKeyEnv != backendValue.APIKeyEnv ||
|
||||
!reflect.DeepEqual(target.ExtraParams, backendValue.ExtraParams) {
|
||||
t.Fatalf("backend defaults were not inherited: %+v", target)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerPrepareBackendResolutionAndCredentialPrecedence(t *testing.T) {
|
||||
resolver := fakeBackendResolver{backends: map[string]domain.Backend{
|
||||
"custom": {ID: "custom", Endpoint: "http://backend/v1", APIKeyEnv: "BACKEND_KEY"},
|
||||
}}
|
||||
promptRepo := &fakePromptRepo{def: promptDef(domain.FormatText, domain.ValidationNone, 0)}
|
||||
|
||||
t.Run("unknown backend is a profile load failure", func(t *testing.T) {
|
||||
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||
"exec": {ID: "exec", BackendID: "unknown", Model: "model"},
|
||||
}}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||
if !errors.Is(err, ErrProfileLoad) {
|
||||
t.Fatalf("expected ErrProfileLoad, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("nil resolver is a profile load failure", func(t *testing.T) {
|
||||
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||
"exec": {ID: "exec", BackendID: "custom", Model: "model"},
|
||||
}}, nil, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||
if !errors.Is(err, ErrProfileLoad) {
|
||||
t.Fatalf("expected ErrProfileLoad, got %v", err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("request environment wins", func(t *testing.T) {
|
||||
t.Setenv("REQUEST_KEY", "request-secret")
|
||||
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||
"exec": {ID: "exec", BackendID: "custom", Model: "model", APIKeyEnv: "PROFILE_KEY"},
|
||||
}}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||
PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(),
|
||||
Execution: &domain.ExecutionTargetOverride{APIKeyEnv: "REQUEST_KEY"},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare: %v", err)
|
||||
}
|
||||
if prepared.EffectiveModelParams.APIKeyEnv != "REQUEST_KEY" {
|
||||
t.Fatalf("unexpected credential source: %+v", prepared.EffectiveModelParams)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("required direct key clears inherited environment", func(t *testing.T) {
|
||||
t.Setenv("BACKEND_KEY", "backend-secret")
|
||||
runner := NewRunner(promptRepo, &fakeExecutionProfileRepo{profiles: map[string]*domain.ExecutionProfile{
|
||||
"exec": {ID: "exec", BackendID: "custom", Model: "model", APIKeyRequired: true},
|
||||
}}, resolver, defaultArtifactReader(), defaultRenderer(), &fakeLLM{forbid: true}, nil)
|
||||
_, err := runner.Prepare(context.Background(), domain.RunRequest{PromptID: "p", ProfileID: "exec", Inputs: singleInputRef()})
|
||||
if !errors.Is(err, ErrAPIKeyRequired) {
|
||||
t.Fatalf("expected ErrAPIKeyRequired, got %v", err)
|
||||
}
|
||||
prepared, err := runner.Prepare(context.Background(), domain.RunRequest{
|
||||
PromptID: "p", ProfileID: "exec", Inputs: singleInputRef(), APIKey: "direct-secret",
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatalf("prepare with direct key: %v", err)
|
||||
}
|
||||
if prepared.EffectiveModelParams.APIKeyEnv != "" {
|
||||
t.Fatalf("expected inherited environment name to be cleared, got %q", prepared.EffectiveModelParams.APIKeyEnv)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func defaultArtifactReader() *fakeArtifactReader {
|
||||
return &fakeArtifactReader{artifactsByURI: map[string]*domain.Artifact{
|
||||
"a://ok": {Body: []byte("x"), Hash: hashString("x")},
|
||||
@@ -1838,10 +1977,11 @@ func intPtr(v int) *int {
|
||||
func newMinimalRunner(promptRepo *fakePromptRepo, execRepo *fakeExecutionProfileRepo) *Runner {
|
||||
return NewRunner(
|
||||
promptRepo,
|
||||
execRepo,
|
||||
execRepo, nil,
|
||||
|
||||
defaultArtifactReader(),
|
||||
defaultRenderer(),
|
||||
&fakeLLM{resp: &domain.GenerateResponse{Content: "ok"}},
|
||||
nil,
|
||||
)
|
||||
nil)
|
||||
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user