package usecase import ( "context" "errors" "testing" "gitea.maximumdirect.net/eric/ConfigKit" "gitea.maximumdirect.net/eric/promptkit/internal/configuration" "gitea.maximumdirect.net/eric/promptkit/internal/domain" "gitea.maximumdirect.net/eric/promptkit/internal/profile" ) type selectionTestCollaborators struct { artifactReads int renderCalls int generationCalls int validationCalls int } type selectionTestArtifactReader struct{ state *selectionTestCollaborators } func (r selectionTestArtifactReader) Read(context.Context, domain.ArtifactRef) (*domain.Artifact, error) { r.state.artifactReads++ return nil, errors.New("artifact access must not occur during target selection") } type selectionTestRenderer struct{ state *selectionTestCollaborators } func (r selectionTestRenderer) Render(context.Context, *domain.PromptDefinition, map[string]*domain.Artifact, map[string]string) (*domain.RenderedPrompt, error) { r.state.renderCalls++ return nil, errors.New("prompt rendering must not occur during target selection") } type selectionTestLLM struct{ state *selectionTestCollaborators } func (c selectionTestLLM) Generate(context.Context, domain.GenerateRequest) (*domain.GenerateResponse, error) { c.state.generationCalls++ return nil, errors.New("generation must not occur during target selection") } type selectionTestValidator struct{ state *selectionTestCollaborators } func (v selectionTestValidator) Validate(context.Context, *domain.Artifact, domain.OutputContract) (domain.ValidationResult, error) { v.state.validationCalls++ return domain.ValidationResult{}, errors.New("validation must not occur during target selection") } type selectionTestRepositories struct { profiles profile.Repository backends smokeBackendLookup profileCalls int } func (r *selectionTestRepositories) GetProfile(ctx context.Context, id string) (*domain.ExecutionProfile, error) { r.profileCalls++ return r.profiles.GetProfile(ctx, id) } func (r *selectionTestRepositories) GetBackend(id string) (domain.Backend, error) { return r.backends.GetBackend(id) } func TestResolveSmokeTestTargetSelectionAndRouting(t *testing.T) { runner, collaborators := newSmokeSelectionRunner(t) for _, test := range []struct { name string request domain.SmokeTestRequest profileID string backendID string selectionSource string defaultSourceBackend string endpoint string settingsEndpoint string wantModel string }{ { name: "explicit profile", request: domain.SmokeTestRequest{Kind: domain.SmokeTestTextInference, ProfileID: "alpha-profile"}, profileID: "alpha-profile", backendID: "alpha", selectionSource: smokeSelectionSourceProfile, endpoint: "https://alpha.example/v1", settingsEndpoint: "https://alpha.example/v1", wantModel: "alpha-model", }, { name: "prompt default", request: domain.SmokeTestRequest{ Kind: domain.SmokeTestTextInference, PromptID: "selection.prompt", }, profileID: "prompt-profile", backendID: "beta", selectionSource: smokeSelectionSourcePrompt, endpoint: "https://beta.example/v1", settingsEndpoint: "https://beta.example/v1", wantModel: "prompt-model", }, { name: "explicit profile overrides prompt default", request: domain.SmokeTestRequest{ Kind: domain.SmokeTestTextInference, PromptID: "selection.prompt", ProfileID: "alpha-profile", BackendID: "beta", }, profileID: "alpha-profile", backendID: "alpha", selectionSource: smokeSelectionSourceProfile, endpoint: "https://alpha.example/v1", settingsEndpoint: "https://alpha.example/v1", wantModel: "alpha-model", }, { name: "backend default routes to a different backend", request: domain.SmokeTestRequest{ Kind: domain.SmokeTestStructuredOutput, BackendID: "alpha", }, profileID: "cross-backend-profile", backendID: "beta", selectionSource: smokeSelectionSourceBackendDefault, defaultSourceBackend: "alpha", endpoint: "https://beta.example/v1", settingsEndpoint: "https://beta.example/v1", wantModel: "cross-model", }, { name: "backend default may select endpoint-only profile", request: domain.SmokeTestRequest{ Kind: domain.SmokeTestTextInference, BackendID: "epsilon", }, profileID: "endpoint-profile", selectionSource: smokeSelectionSourceBackendDefault, defaultSourceBackend: "epsilon", endpoint: "https://endpoint-only.example/v1", settingsEndpoint: "https://endpoint-only.example/v1", wantModel: "endpoint-model", }, { name: "prompt profile supplies backend-only target over backend hint", request: domain.SmokeTestRequest{ Kind: domain.SmokeTestReachability, PromptID: "selection.prompt", BackendID: "alpha", }, profileID: "prompt-profile", backendID: "beta", selectionSource: smokeSelectionSourcePrompt, endpoint: "https://beta.example/v1", settingsEndpoint: "https://beta.example/v1", wantModel: "prompt-model", }, { name: "backend-only probe uses backend when prompt has no profile default", request: domain.SmokeTestRequest{ Kind: domain.SmokeTestReachability, PromptID: "backend.prompt", BackendID: "delta", }, backendID: "delta", selectionSource: smokeSelectionSourceBackend, endpoint: "https://delta.example/v1", settingsEndpoint: "https://delta.example/v1", }, } { t.Run(test.name, func(t *testing.T) { selected, err := runner.resolveSmokeTestTarget(context.Background(), test.request) if err != nil { t.Fatalf("resolve target: %v", err) } if selected.target.ProfileID != test.profileID || selected.target.BackendID != test.backendID || selected.target.SelectionSource != test.selectionSource || selected.target.DefaultSourceBackendID != test.defaultSourceBackend || selected.target.Endpoint != test.endpoint || selected.target.EffectiveSettings.Endpoint != test.settingsEndpoint || selected.target.EffectiveSettings.Model != test.wantModel { t.Fatalf("unexpected target: %+v", selected.target) } if test.request.PromptID != "" && selected.target.PromptID != test.request.PromptID { t.Fatalf("prompt identity was not retained: %+v", selected.target) } }) } if collaborators.artifactReads != 0 || collaborators.renderCalls != 0 || collaborators.generationCalls != 0 || collaborators.validationCalls != 0 { t.Fatalf("selection entered application preparation: %+v", collaborators) } } func TestResolveSmokeTestTargetBackendOnlyDoesNotLoadDefaultProfile(t *testing.T) { loaded, err := configuration.Load(configkit.Input{Source: configkit.Mapping("selection", smokeSelectionConfig())}, nil) if err != nil { t.Fatal(err) } repositories := &selectionTestRepositories{profiles: loaded, backends: loaded} runner := NewRunner(nil, repositories, nil, nil, nil, nil, nil) zeroTemperature := 0.0 selected, err := runner.resolveSmokeTestTarget(context.Background(), domain.SmokeTestRequest{ Kind: domain.SmokeTestReachability, BackendID: "delta", Execution: &domain.ExecutionTargetOverride{Temperature: &zeroTemperature}, }) if err != nil { t.Fatalf("backend-only probe should not resolve its dangling default profile: %v", err) } if selected.target.SelectionSource != smokeSelectionSourceBackend || selected.target.BackendID != "delta" || selected.target.ProfileID != "" || selected.target.Endpoint != "https://delta.example/v1" { t.Fatalf("unexpected direct backend target: %+v", selected.target) } if !selected.target.SettingsPresence.Temperature || selected.target.EffectiveSettings.Temperature != zeroTemperature { t.Fatalf("explicit zero override presence was not retained: %+v", selected.target) } if repositories.profileCalls != 0 { t.Fatalf("backend-only selection loaded %d profiles", repositories.profileCalls) } } func TestResolveSmokeTestTargetPreservesDiagnosticAndBaseEndpoints(t *testing.T) { runner, _ := newSmokeSelectionRunner(t) override := domain.ExecutionTargetOverride{Endpoint: " https://override.example/v1 "} selected, err := runner.resolveSmokeTestTarget(context.Background(), domain.SmokeTestRequest{ Kind: domain.SmokeTestReachability, ProfileID: "incomplete-profile", Execution: &override, }) if err != nil { t.Fatalf("HTTP diagnostic should not require a model: %v", err) } if selected.target.BackendID != "alpha" || selected.target.EffectiveSettings.Model != "" || selected.target.EffectiveSettings.Endpoint != "https://override.example/v1" || selected.target.Endpoint != "https://probe.example/ready?check=1" { t.Fatalf("diagnostic destination and base settings were not kept separately: %+v", selected.target) } selected, err = runner.resolveSmokeTestTarget(context.Background(), domain.SmokeTestRequest{ Kind: domain.SmokeTestAuthentication, BackendID: "beta", }) if err != nil { t.Fatal(err) } if selected.target.Endpoint != "https://beta.example/v1/key" { t.Fatalf("authentication destination = %q", selected.target.Endpoint) } } func TestResolveSmokeTestTargetSelectionAndSettingsErrors(t *testing.T) { runner, _ := newSmokeSelectionRunner(t) negativeRetries := -1 negativeTimeout := -1 for _, test := range []struct { name string request domain.SmokeTestRequest wantErr error }{ { name: "unknown kind", request: domain.SmokeTestRequest{Kind: "unknown", BackendID: "epsilon"}, wantErr: ErrInvalidRequest, }, { name: "negative retry override", request: domain.SmokeTestRequest{Kind: domain.SmokeTestReachability, BackendID: "epsilon", TransientRetries: &negativeRetries}, wantErr: ErrInvalidRequest, }, { name: "explicit selection does not fall back", request: domain.SmokeTestRequest{ Kind: domain.SmokeTestTextInference, ProfileID: "missing-profile", BackendID: "epsilon", }, wantErr: ErrProfileLoad, }, { name: "invalid prompt selection does not fall back", request: domain.SmokeTestRequest{ Kind: domain.SmokeTestTextInference, PromptID: "missing.prompt", BackendID: "epsilon", }, wantErr: ErrPromptLoad, }, { name: "blank explicit profile does not fall back", request: domain.SmokeTestRequest{ Kind: domain.SmokeTestTextInference, ProfileID: " ", BackendID: "epsilon", }, wantErr: ErrInvalidRequest, }, { name: "dangling backend default fails only when selected", request: domain.SmokeTestRequest{Kind: domain.SmokeTestTextInference, BackendID: "delta"}, wantErr: ErrProfileLoad, }, { name: "unused negative setting is still validated for HTTP", request: domain.SmokeTestRequest{ Kind: domain.SmokeTestReachability, BackendID: "epsilon", Execution: &domain.ExecutionTargetOverride{TimeoutSeconds: &negativeTimeout}, }, wantErr: ErrInvalidRequest, }, } { t.Run(test.name, func(t *testing.T) { if _, err := runner.resolveSmokeTestTarget(context.Background(), test.request); !errors.Is(err, test.wantErr) { t.Fatalf("selection error = %v, want errors.Is(_, %v)", err, test.wantErr) } }) } } func newSmokeSelectionRunner(t *testing.T) (*Runner, *selectionTestCollaborators) { t.Helper() loaded, err := configuration.Load(configkit.Input{Source: configkit.Mapping("selection", smokeSelectionConfig())}, nil) if err != nil { t.Fatalf("load captured selection configuration: %v", err) } collaborators := &selectionTestCollaborators{} return NewRunner( loaded, loaded, selectionTestArtifactReader{state: collaborators}, selectionTestRenderer{state: collaborators}, selectionTestLLM{state: collaborators}, selectionTestValidator{state: collaborators}, nil, ), collaborators } func smokeSelectionConfig() map[string]any { return map[string]any{ "backends": map[string]any{ "alpha": map[string]any{ "endpoint": "https://alpha.example/v1", "smoke_tests": map[string]any{ "default_profile": "cross-backend-profile", "reachability": map[string]any{"endpoint": "https://probe.example/ready?check=1"}, "authentication": map[string]any{"kind": "openrouter_key"}, }, }, "beta": map[string]any{ "endpoint": "https://beta.example/v1", "smoke_tests": map[string]any{"authentication": map[string]any{"kind": "openrouter_key"}}, }, "delta": map[string]any{ "endpoint": "https://delta.example/v1", "smoke_tests": map[string]any{"default_profile": "not-yet-defined"}, }, "epsilon": map[string]any{ "endpoint": "https://epsilon.example/v1", "smoke_tests": map[string]any{"default_profile": "endpoint-profile"}, }, }, "profiles": map[string]any{ "alpha-profile": map[string]any{"backend": "alpha", "model": "alpha-model"}, "prompt-profile": map[string]any{"backend": "beta", "model": "prompt-model"}, "cross-backend-profile": map[string]any{"backend": "beta", "model": "cross-model"}, "endpoint-profile": map[string]any{"endpoint": "https://endpoint-only.example/v1", "model": "endpoint-model"}, "incomplete-profile": map[string]any{"backend": "alpha"}, }, "prompts": map[string]any{ "selection.prompt": map[string]any{ "version": "v1", "default_profile": "prompt-profile", "inputs": []any{map[string]any{"name": "required_input", "required": true}}, "messages": []any{map[string]any{"role": "user", "content": "Hello {{.required_input}}"}}, "session_id": "{{.vars.session}}", "output": map[string]any{ "format": "json", "validation_mode": "json_schema", "schema_path": "not-read.json", }, }, "backend.prompt": map[string]any{ "version": "v1", "messages": []any{map[string]any{"role": "user", "content": "Hello"}}, "output": map[string]any{ "format": "json", "validation_mode": "json_schema", "schema_path": "not-read.json", }, }, }, } }