From 27ff580dae097af6aeb3f29d2e13b31353e687d7 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Tue, 13 Oct 2026 16:14:30 +0000 Subject: [PATCH] Resolve smoke-test targets --- docs/roadmap/implementation.md | 3 + internal/usecase/profile_inspection.go | 14 +- internal/usecase/smoke_selection.go | 193 ++++++++++++ internal/usecase/smoke_selection_test.go | 358 +++++++++++++++++++++++ 4 files changed, 565 insertions(+), 3 deletions(-) create mode 100644 internal/usecase/smoke_selection.go create mode 100644 internal/usecase/smoke_selection_test.go diff --git a/docs/roadmap/implementation.md b/docs/roadmap/implementation.md index f25f396..c83b2b4 100644 --- a/docs/roadmap/implementation.md +++ b/docs/roadmap/implementation.md @@ -289,6 +289,9 @@ selection failure, default-reference failure only on use, cross-backend/default identity, endpoint-only profiles, and HTTP model incompleteness. Use real captured configuration and failing external readers to prove no application preparation. +**Completed: 2026-10-13.** Validation passed: `go test ./...`, `go vet ./...`, +and `git diff --check`. + ## Stage 4: Add OpenRouter Catalog Diagnostic Assets Scope: `../promptkit-backend-openrouter` only, plus this plan's completion record. diff --git a/internal/usecase/profile_inspection.go b/internal/usecase/profile_inspection.go index e1c3a56..be7bd74 100644 --- a/internal/usecase/profile_inspection.go +++ b/internal/usecase/profile_inspection.go @@ -76,14 +76,22 @@ func (r *Runner) resolveProfileIdentity(ctx context.Context, profileID string) ( } func normalizeResolvedExecutionTarget(target domain.ExecutionTarget) (domain.ExecutionTarget, error) { + target, err := normalizeResolvedExecutionSettings(target) + if err != nil { + return domain.ExecutionTarget{}, err + } + if strings.TrimSpace(target.Model) == "" { + return domain.ExecutionTarget{}, errors.New("execution model is required") + } + return target, nil +} + +func normalizeResolvedExecutionSettings(target domain.ExecutionTarget) (domain.ExecutionTarget, error) { endpoint, err := domain.NormalizeOpenAICompatibleBaseEndpoint(target.Endpoint) if err != nil { return domain.ExecutionTarget{}, fmt.Errorf("execution endpoint: %w", err) } target.Endpoint = endpoint - if strings.TrimSpace(target.Model) == "" { - return domain.ExecutionTarget{}, errors.New("execution model is required") - } if err := domain.ValidateExecutionTargetSettings(target); err != nil { return domain.ExecutionTarget{}, err } diff --git a/internal/usecase/smoke_selection.go b/internal/usecase/smoke_selection.go new file mode 100644 index 0000000..0bd91df --- /dev/null +++ b/internal/usecase/smoke_selection.go @@ -0,0 +1,193 @@ +package usecase + +import ( + "context" + "fmt" + "net/url" + "strings" + + "gitea.maximumdirect.net/eric/promptkit/internal/defaults" + "gitea.maximumdirect.net/eric/promptkit/internal/domain" +) + +const ( + smokeSelectionSourceBackendDefault = "backend_default" + smokeSelectionSourceProfile = "profile" + smokeSelectionSourcePrompt = "prompt" + smokeSelectionSourceBackend = "backend" +) + +type smokeBackendLookup interface { + GetBackend(string) (domain.Backend, error) +} + +type smokeTestSelection struct { + target domain.SmokeTestTarget + backend domain.Backend + transientRetries uint64 +} + +func (r *Runner) resolveSmokeTestTarget(ctx context.Context, req domain.SmokeTestRequest) (*smokeTestSelection, error) { + if !validSmokeTestKind(req.Kind) { + return nil, fmt.Errorf("%w: unknown smoke-test kind %q", ErrInvalidRequest, req.Kind) + } + transientRetries, err := domain.ResolveTransientRetries(0, req.TransientRetries) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err) + } + + promptID := strings.TrimSpace(req.PromptID) + profileID := strings.TrimSpace(req.ProfileID) + backendID := strings.TrimSpace(req.BackendID) + if promptID == "" && strings.TrimSpace(req.PromptVersion) != "" { + return nil, fmt.Errorf("%w: prompt version requires a prompt id", ErrInvalidRequest) + } + + var promptInspection *domain.PromptInspection + if promptID != "" { + promptInspection, err = r.resolvePromptInspection(ctx, req.PromptID, req.PromptVersion) + if err != nil { + return nil, err + } + } + + selection := &smokeTestSelection{transientRetries: transientRetries} + var selectedProfileID string + switch { + case profileID != "": + selectedProfileID = req.ProfileID + selection.target.SelectionSource = smokeSelectionSourceProfile + case promptInspection != nil && (promptInspection.DefaultProfileID != "" || isInferenceSmokeTest(req.Kind)): + selectedProfileID = promptInspection.DefaultProfileID + selection.target.SelectionSource = smokeSelectionSourcePrompt + case backendID != "" && isInferenceSmokeTest(req.Kind): + backend, err := r.resolveSmokeBackend(req.BackendID) + if err != nil { + return nil, err + } + if backend.SmokeTests == nil || backend.SmokeTests.DefaultProfile == nil { + return nil, fmt.Errorf("%w: backend %q has no default smoke-test profile", ErrProfileRequired, req.BackendID) + } + selectedProfileID = *backend.SmokeTests.DefaultProfile + selection.backend = backend + selection.target.DefaultSourceBackendID = backend.ID + selection.target.SelectionSource = smokeSelectionSourceBackendDefault + case backendID != "": + backend, err := r.resolveSmokeBackend(req.BackendID) + if err != nil { + return nil, err + } + selection.backend = backend + selection.target.SelectionSource = smokeSelectionSourceBackend + default: + if isInferenceSmokeTest(req.Kind) { + return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, ErrProfileRequired) + } + return nil, fmt.Errorf("%w: backend id, profile id, or prompt id is required", ErrInvalidRequest) + } + if selection.target.SelectionSource != smokeSelectionSourceBackend && selectedProfileID == "" { + return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, ErrProfileRequired) + } + + if selectedProfileID != "" { + selection.backend = domain.Backend{} + profileSelection, err := r.resolveProfileSelection(ctx, selectedProfileID) + if err != nil { + return nil, err + } + settings, presence := resolveExecutionTarget(profileSelection.profile, req.Execution) + settings.APIKey = req.APIKey + if isInferenceSmokeTest(req.Kind) { + settings, err = normalizeResolvedExecutionTarget(settings) + } else { + settings, err = normalizeResolvedExecutionSettings(settings) + } + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err) + } + selection.target.ProfileID = profileSelection.id + selection.target.BackendID = settings.BackendID + selection.target.EffectiveSettings = settings + selection.target.SettingsPresence = presence + if settings.BackendID != "" { + selection.backend, err = r.resolveSmokeBackend(settings.BackendID) + if err != nil { + return nil, err + } + } + } else { + settings := defaults.ExecutionTargetDefault() + settings.BackendID = selection.backend.ID + settings.Endpoint = selection.backend.Endpoint + settings.APIKeyEnv = selection.backend.APIKeyEnv + settings.ExtraParams = copyExtraParams(selection.backend.ExtraParams) + presence := domain.ExecutionTargetPresence{} + if req.Execution != nil { + settings, presence = mergeExecutionTargetOverride(settings, presence, *req.Execution) + } + settings.APIKey = req.APIKey + settings, err = normalizeResolvedExecutionSettings(settings) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrInvalidRequest, err) + } + selection.target.BackendID = settings.BackendID + selection.target.EffectiveSettings = settings + selection.target.SettingsPresence = presence + } + + if promptInspection != nil { + selection.target.PromptID = promptInspection.PromptID + selection.target.PromptVersion = promptInspection.PromptVersion + } + selection.target.Endpoint, err = resolveSmokeTestEndpoint(req.Kind, selection.target.EffectiveSettings.Endpoint, selection.backend.SmokeTests) + if err != nil { + return nil, fmt.Errorf("%w: diagnostic endpoint: %w", ErrInvalidRequest, err) + } + return selection, nil +} + +func (r *Runner) resolveSmokeBackend(id string) (domain.Backend, error) { + lookup, ok := r.profiles.(smokeBackendLookup) + if !ok { + return domain.Backend{}, fmt.Errorf("%w: backend lookup is not configured", ErrProfileLoad) + } + backend, err := lookup.GetBackend(id) + if err != nil { + return domain.Backend{}, fmt.Errorf("%w: backend %q: %v", ErrProfileLoad, id, err) + } + return backend, nil +} + +func validSmokeTestKind(kind domain.SmokeTestKind) bool { + switch kind { + case domain.SmokeTestReachability, domain.SmokeTestAuthentication, + domain.SmokeTestTextInference, domain.SmokeTestStructuredOutput: + return true + default: + return false + } +} + +func isInferenceSmokeTest(kind domain.SmokeTestKind) bool { + return kind == domain.SmokeTestTextInference || kind == domain.SmokeTestStructuredOutput +} + +func resolveSmokeTestEndpoint(kind domain.SmokeTestKind, baseEndpoint string, smokeTests *domain.BackendSmokeTestConfig) (string, error) { + switch kind { + case domain.SmokeTestReachability: + if smokeTests != nil && smokeTests.Reachability != nil && smokeTests.Reachability.Endpoint != nil { + return *smokeTests.Reachability.Endpoint, nil + } + case domain.SmokeTestAuthentication: + if smokeTests != nil && smokeTests.Authentication != nil { + authentication := smokeTests.Authentication + if authentication.Endpoint != nil { + return *authentication.Endpoint, nil + } + if authentication.Kind != nil && *authentication.Kind == "openrouter_key" { + return url.JoinPath(baseEndpoint, "key") + } + } + } + return baseEndpoint, nil +} diff --git a/internal/usecase/smoke_selection_test.go b/internal/usecase/smoke_selection_test.go new file mode 100644 index 0000000..c6becf1 --- /dev/null +++ b/internal/usecase/smoke_selection_test.go @@ -0,0 +1,358 @@ +package usecase + +import ( + "context" + "errors" + "testing" + + "gitea.maximumdirect.net/eric/ConfigKit" + "gitea.maximumdirect.net/eric/promptkit/internal/configuration" + "gitea.maximumdirect.net/eric/promptkit/internal/domain" + "gitea.maximumdirect.net/eric/promptkit/internal/profile" +) + +type selectionTestCollaborators struct { + artifactReads int + renderCalls int + generationCalls int + validationCalls int +} + +type selectionTestArtifactReader struct{ state *selectionTestCollaborators } + +func (r selectionTestArtifactReader) Read(context.Context, domain.ArtifactRef) (*domain.Artifact, error) { + r.state.artifactReads++ + return nil, errors.New("artifact access must not occur during target selection") +} + +type selectionTestRenderer struct{ state *selectionTestCollaborators } + +func (r selectionTestRenderer) Render(context.Context, *domain.PromptDefinition, map[string]*domain.Artifact, map[string]string) (*domain.RenderedPrompt, error) { + r.state.renderCalls++ + return nil, errors.New("prompt rendering must not occur during target selection") +} + +type selectionTestLLM struct{ state *selectionTestCollaborators } + +func (c selectionTestLLM) Generate(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error) { + c.state.generationCalls++ + return nil, errors.New("generation must not occur during target selection") +} + +type selectionTestValidator struct{ state *selectionTestCollaborators } + +func (v selectionTestValidator) Validate(context.Context, *domain.Artifact, domain.OutputContract) (domain.ValidationResult, error) { + v.state.validationCalls++ + return domain.ValidationResult{}, errors.New("validation must not occur during target selection") +} + +type selectionTestRepositories struct { + profiles profile.Repository + backends smokeBackendLookup + profileCalls int +} + +func (r *selectionTestRepositories) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) { + r.profileCalls++ + return r.profiles.GetProfile(ctx, id) +} + +func (r *selectionTestRepositories) GetBackend(id string) (domain.Backend, error) { + return r.backends.GetBackend(id) +} + +func TestResolveSmokeTestTargetSelectionAndRouting(t *testing.T) { + runner, collaborators := newSmokeSelectionRunner(t) + + for _, test := range []struct { + name string + request domain.SmokeTestRequest + profileID string + backendID string + selectionSource string + defaultSourceBackend string + endpoint string + settingsEndpoint string + wantModel string + }{ + { + name: "explicit profile", + request: domain.SmokeTestRequest{Kind: domain.SmokeTestTextInference, ProfileID: "alpha-profile"}, + profileID: "alpha-profile", + backendID: "alpha", + selectionSource: smokeSelectionSourceProfile, + endpoint: "https://alpha.example/v1", + settingsEndpoint: "https://alpha.example/v1", + wantModel: "alpha-model", + }, + { + name: "prompt default", + request: domain.SmokeTestRequest{ + Kind: domain.SmokeTestTextInference, PromptID: "selection.prompt", PromptVersion: "v1", + }, + profileID: "prompt-profile", + backendID: "beta", + selectionSource: smokeSelectionSourcePrompt, + endpoint: "https://beta.example/v1", + settingsEndpoint: "https://beta.example/v1", + wantModel: "prompt-model", + }, + { + name: "explicit profile overrides prompt default", + request: domain.SmokeTestRequest{ + Kind: domain.SmokeTestTextInference, PromptID: "selection.prompt", PromptVersion: "v1", + ProfileID: "alpha-profile", BackendID: "beta", + }, + profileID: "alpha-profile", + backendID: "alpha", + selectionSource: smokeSelectionSourceProfile, + endpoint: "https://alpha.example/v1", + settingsEndpoint: "https://alpha.example/v1", + wantModel: "alpha-model", + }, + { + name: "backend default routes to a different backend", + request: domain.SmokeTestRequest{ + Kind: domain.SmokeTestStructuredOutput, BackendID: "alpha", + }, + profileID: "cross-backend-profile", + backendID: "beta", + selectionSource: smokeSelectionSourceBackendDefault, + defaultSourceBackend: "alpha", + endpoint: "https://beta.example/v1", + settingsEndpoint: "https://beta.example/v1", + wantModel: "cross-model", + }, + { + name: "backend default may select endpoint-only profile", + request: domain.SmokeTestRequest{ + Kind: domain.SmokeTestTextInference, BackendID: "epsilon", + }, + profileID: "endpoint-profile", + selectionSource: smokeSelectionSourceBackendDefault, + defaultSourceBackend: "epsilon", + endpoint: "https://endpoint-only.example/v1", + settingsEndpoint: "https://endpoint-only.example/v1", + wantModel: "endpoint-model", + }, + { + name: "prompt profile supplies backend-only target over backend hint", + request: domain.SmokeTestRequest{ + Kind: domain.SmokeTestReachability, PromptID: "selection.prompt", PromptVersion: "v1", BackendID: "alpha", + }, + profileID: "prompt-profile", + backendID: "beta", + selectionSource: smokeSelectionSourcePrompt, + endpoint: "https://beta.example/v1", + settingsEndpoint: "https://beta.example/v1", + wantModel: "prompt-model", + }, + { + name: "backend-only probe uses backend when prompt has no profile default", + request: domain.SmokeTestRequest{ + Kind: domain.SmokeTestReachability, PromptID: "backend.prompt", PromptVersion: "v1", BackendID: "delta", + }, + backendID: "delta", + selectionSource: smokeSelectionSourceBackend, + endpoint: "https://delta.example/v1", + settingsEndpoint: "https://delta.example/v1", + }, + } { + t.Run(test.name, func(t *testing.T) { + selected, err := runner.resolveSmokeTestTarget(context.Background(), test.request) + if err != nil { + t.Fatalf("resolve target: %v", err) + } + if selected.target.ProfileID != test.profileID || selected.target.BackendID != test.backendID || + selected.target.SelectionSource != test.selectionSource || + selected.target.DefaultSourceBackendID != test.defaultSourceBackend || + selected.target.Endpoint != test.endpoint || + selected.target.EffectiveSettings.Endpoint != test.settingsEndpoint || + selected.target.EffectiveSettings.Model != test.wantModel { + t.Fatalf("unexpected target: %+v", selected.target) + } + if test.request.PromptID != "" && (selected.target.PromptID != test.request.PromptID || selected.target.PromptVersion != test.request.PromptVersion) { + t.Fatalf("prompt identity was not retained: %+v", selected.target) + } + }) + } + + if collaborators.artifactReads != 0 || collaborators.renderCalls != 0 || + collaborators.generationCalls != 0 || collaborators.validationCalls != 0 { + t.Fatalf("selection entered application preparation: %+v", collaborators) + } +} + +func TestResolveSmokeTestTargetBackendOnlyDoesNotLoadDefaultProfile(t *testing.T) { + loaded, err := configuration.Load(configkit.Input{Source: configkit.Mapping("selection", smokeSelectionConfig())}, nil) + if err != nil { + t.Fatal(err) + } + repositories := &selectionTestRepositories{profiles: loaded, backends: loaded} + runner := NewRunner(nil, repositories, nil, nil, nil, nil, nil) + + zeroTemperature := 0.0 + selected, err := runner.resolveSmokeTestTarget(context.Background(), domain.SmokeTestRequest{ + Kind: domain.SmokeTestReachability, BackendID: "delta", + Execution: &domain.ExecutionTargetOverride{Temperature: &zeroTemperature}, + }) + if err != nil { + t.Fatalf("backend-only probe should not resolve its dangling default profile: %v", err) + } + if selected.target.SelectionSource != smokeSelectionSourceBackend || selected.target.BackendID != "delta" || + selected.target.ProfileID != "" || selected.target.Endpoint != "https://delta.example/v1" { + t.Fatalf("unexpected direct backend target: %+v", selected.target) + } + if !selected.target.SettingsPresence.Temperature || selected.target.EffectiveSettings.Temperature != zeroTemperature { + t.Fatalf("explicit zero override presence was not retained: %+v", selected.target) + } + if repositories.profileCalls != 0 { + t.Fatalf("backend-only selection loaded %d profiles", repositories.profileCalls) + } +} + +func TestResolveSmokeTestTargetPreservesDiagnosticAndBaseEndpoints(t *testing.T) { + runner, _ := newSmokeSelectionRunner(t) + override := domain.ExecutionTargetOverride{Endpoint: " https://override.example/v1 "} + selected, err := runner.resolveSmokeTestTarget(context.Background(), domain.SmokeTestRequest{ + Kind: domain.SmokeTestReachability, ProfileID: "incomplete-profile", Execution: &override, + }) + if err != nil { + t.Fatalf("HTTP diagnostic should not require a model: %v", err) + } + if selected.target.BackendID != "alpha" || selected.target.EffectiveSettings.Model != "" || selected.target.EffectiveSettings.Endpoint != "https://override.example/v1" || + selected.target.Endpoint != "https://probe.example/ready?check=1" { + t.Fatalf("diagnostic destination and base settings were not kept separately: %+v", selected.target) + } + + selected, err = runner.resolveSmokeTestTarget(context.Background(), domain.SmokeTestRequest{ + Kind: domain.SmokeTestAuthentication, BackendID: "beta", + }) + if err != nil { + t.Fatal(err) + } + if selected.target.Endpoint != "https://beta.example/v1/key" { + t.Fatalf("authentication destination = %q", selected.target.Endpoint) + } +} + +func TestResolveSmokeTestTargetSelectionAndSettingsErrors(t *testing.T) { + runner, _ := newSmokeSelectionRunner(t) + negativeRetries := -1 + negativeTimeout := -1 + for _, test := range []struct { + name string + request domain.SmokeTestRequest + wantErr error + }{ + { + name: "unknown kind", + request: domain.SmokeTestRequest{Kind: "unknown", BackendID: "epsilon"}, + wantErr: ErrInvalidRequest, + }, + { + name: "negative retry override", + request: domain.SmokeTestRequest{Kind: domain.SmokeTestReachability, BackendID: "epsilon", TransientRetries: &negativeRetries}, + wantErr: ErrInvalidRequest, + }, + { + name: "explicit selection does not fall back", + request: domain.SmokeTestRequest{ + Kind: domain.SmokeTestTextInference, ProfileID: "missing-profile", BackendID: "epsilon", + }, + wantErr: ErrProfileLoad, + }, + { + name: "dangling backend default fails only when selected", + request: domain.SmokeTestRequest{Kind: domain.SmokeTestTextInference, BackendID: "delta"}, + wantErr: ErrProfileLoad, + }, + { + name: "unused negative setting is still validated for HTTP", + request: domain.SmokeTestRequest{ + Kind: domain.SmokeTestReachability, BackendID: "epsilon", + Execution: &domain.ExecutionTargetOverride{TimeoutSeconds: &negativeTimeout}, + }, + wantErr: ErrInvalidRequest, + }, + } { + t.Run(test.name, func(t *testing.T) { + if _, err := runner.resolveSmokeTestTarget(context.Background(), test.request); !errors.Is(err, test.wantErr) { + t.Fatalf("selection error = %v, want errors.Is(_, %v)", err, test.wantErr) + } + }) + } +} + +func newSmokeSelectionRunner(t *testing.T) (*Runner, *selectionTestCollaborators) { + t.Helper() + loaded, err := configuration.Load(configkit.Input{Source: configkit.Mapping("selection", smokeSelectionConfig())}, nil) + if err != nil { + t.Fatalf("load captured selection configuration: %v", err) + } + collaborators := &selectionTestCollaborators{} + return NewRunner( + loaded, + loaded, + selectionTestArtifactReader{state: collaborators}, + selectionTestRenderer{state: collaborators}, + selectionTestLLM{state: collaborators}, + selectionTestValidator{state: collaborators}, + nil, + ), collaborators +} + +func smokeSelectionConfig() map[string]any { + return map[string]any{ + "backends": map[string]any{ + "alpha": map[string]any{ + "endpoint": "https://alpha.example/v1", + "smoke_tests": map[string]any{ + "default_profile": "cross-backend-profile", + "reachability": map[string]any{"endpoint": "https://probe.example/ready?check=1"}, + "authentication": map[string]any{"kind": "openrouter_key"}, + }, + }, + "beta": map[string]any{ + "endpoint": "https://beta.example/v1", + "smoke_tests": map[string]any{"authentication": map[string]any{"kind": "openrouter_key"}}, + }, + "delta": map[string]any{ + "endpoint": "https://delta.example/v1", + "smoke_tests": map[string]any{"default_profile": "not-yet-defined"}, + }, + "epsilon": map[string]any{ + "endpoint": "https://epsilon.example/v1", + "smoke_tests": map[string]any{"default_profile": "endpoint-profile"}, + }, + }, + "profiles": map[string]any{ + "alpha-profile": map[string]any{"backend": "alpha", "model": "alpha-model"}, + "prompt-profile": map[string]any{"backend": "beta", "model": "prompt-model"}, + "cross-backend-profile": map[string]any{"backend": "beta", "model": "cross-model"}, + "endpoint-profile": map[string]any{"endpoint": "https://endpoint-only.example/v1", "model": "endpoint-model"}, + "incomplete-profile": map[string]any{"backend": "alpha"}, + }, + "prompts": map[string]any{ + "selection.prompt": map[string]any{ + "v1": map[string]any{ + "default_profile": "prompt-profile", + "inputs": []any{map[string]any{"name": "required_input", "required": true}}, + "messages": []any{map[string]any{"role": "user", "content": "Hello {{.required_input}}"}}, + "session_id": "{{.vars.session}}", + "output": map[string]any{ + "format": "json", "validation_mode": "json_schema", "schema_path": "not-read.json", + }, + }, + }, + "backend.prompt": map[string]any{ + "v1": map[string]any{ + "messages": []any{map[string]any{"role": "user", "content": "Hello"}}, + "output": map[string]any{ + "format": "json", "validation_mode": "json_schema", "schema_path": "not-read.json", + }, + }, + }, + }, + } +}