package app import ( "context" "errors" "reflect" "strings" "testing" "time" "gitea.maximumdirect.net/eric/weatherreporter/internal/config" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" "gitea.maximumdirect.net/eric/weatherreporter/internal/report" ) func TestInspectPromptExecutionSelectsDefaultAndOverrideProfiles(t *testing.T) { resolved := inspectionResolved(t) executor := &inspectionExecutor{ prompt: validPromptInspection(resolved.Definition), profiles: map[string]promptexec.ProfileInspection{ "default-profile": {ProfileID: "default-profile", BackendID: "local", ModelName: "default-model"}, "override-profile": {ProfileID: "override-profile", BackendID: "cloud", ModelName: "override-model"}, }, } defaultResult, err := InspectPromptExecution(context.Background(), PromptInspectionRequest{Resolved: resolved, Executor: executor}) if err != nil { t.Fatalf("InspectPromptExecution(default) error = %v", err) } if defaultResult.ProfileID != "default-profile" || defaultResult.ModelName != "default-model" { t.Fatalf("default result = %#v", defaultResult) } overrideResult, err := InspectPromptExecution(context.Background(), PromptInspectionRequest{ Resolved: resolved, Executor: executor, Promptkit: config.PromptkitConfig{Profile: "override-profile"}, }) if err != nil { t.Fatalf("InspectPromptExecution(override) error = %v", err) } if overrideResult.ProfileID != "override-profile" || overrideResult.ModelName != "override-model" { t.Fatalf("override result = %#v", overrideResult) } if len(executor.promptRequests) != 2 || executor.promptRequests[0].version != resolved.Definition.PromptVersion || executor.profileRequests[0] != "default-profile" || executor.profileRequests[1] != "override-profile" { t.Fatalf("inspection requests = prompts %#v profiles %#v", executor.promptRequests, executor.profileRequests) } } func TestInspectPromptExecutionRejectsInvalidContractsAndCredentials(t *testing.T) { resolved := inspectionResolved(t) basePrompt := validPromptInspection(resolved.Definition) tests := []struct { name string prompt promptexec.PromptInspection profile promptexec.ProfileInspection lookupEnv func(string) (string, bool) wantCategory promptexec.ErrorCategory }{ { name: "extra input", prompt: func() promptexec.PromptInspection { value := basePrompt value.Inputs = append(value.Inputs, promptexec.InputDefinition{Name: "unexpected"}) return value }(), wantCategory: promptexec.InvalidConfiguration, }, { name: "wrong schema", prompt: func() promptexec.PromptInspection { value := basePrompt value.Output.SchemaPath = "unexpected.schema.json" return value }(), wantCategory: promptexec.InvalidConfiguration, }, { name: "direct key", prompt: basePrompt, profile: promptexec.ProfileInspection{ProfileID: "default-profile", CredentialRequired: true}, wantCategory: promptexec.MissingCredential, }, { name: "missing environment credential", prompt: basePrompt, profile: promptexec.ProfileInspection{ProfileID: "default-profile", APIKeyEnv: "PROMPT_API_KEY"}, lookupEnv: func(string) (string, bool) { return "", false }, wantCategory: promptexec.MissingCredential, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { executor := &inspectionExecutor{prompt: test.prompt, profiles: map[string]promptexec.ProfileInspection{"default-profile": test.profile}} _, err := InspectPromptExecution(context.Background(), PromptInspectionRequest{Resolved: resolved, Executor: executor, LookupEnv: test.lookupEnv}) if err == nil || promptexec.CategoryOf(err) != test.wantCategory { t.Fatalf("error/category = %v/%q, want %q", err, promptexec.CategoryOf(err), test.wantCategory) } }) } } func TestInspectPromptExecutionReturnsSafeInspectionError(t *testing.T) { resolved := inspectionResolved(t) executor := &inspectionExecutor{promptErr: errors.New("provider response contains resolved-secret-value")} _, err := InspectPromptExecution(context.Background(), PromptInspectionRequest{Resolved: resolved, Executor: executor}) if err == nil || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration { t.Fatalf("error/category = %v/%q", err, promptexec.CategoryOf(err)) } if strings.Contains(err.Error(), "resolved-secret-value") { t.Fatalf("inspection error leaks provider value: %v", err) } } func TestInspectPromptExecutionsReusesEffectiveProfile(t *testing.T) { first := inspectionResolved(t) second := first second.Definition.ID = report.Today second.Definition.PromptID = "weather.today" executor := &inspectionExecutor{ prompt: validPromptInspection(first.Definition), profiles: map[string]promptexec.ProfileInspection{ "default-profile": {ProfileID: "default-profile", BackendID: "local", ModelName: "model"}, }, } executor.prompts = map[string]promptexec.PromptInspection{ first.Definition.PromptID: validPromptInspection(first.Definition), second.Definition.PromptID: validPromptInspection(second.Definition), } results, err := InspectPromptExecutions(context.Background(), PromptExecutionsInspectionRequest{Resolved: []report.Resolved{first, second}, Executor: executor}) if err != nil { t.Fatalf("InspectPromptExecutions() error = %v", err) } if len(results) != 2 || len(executor.profileRequests) != 1 { t.Fatalf("results/profile requests = %#v/%#v, want two results and one profile inspection", results, executor.profileRequests) } } func TestInspectComparisonExecutionPreservesOrderedExplicitProfiles(t *testing.T) { resolved := inspectionResolved(t) executor := &inspectionExecutor{ prompt: validPromptInspection(resolved.Definition), profiles: map[string]promptexec.ProfileInspection{ "weather-light": {ProfileID: "weather-light", BackendID: "local", ModelName: "light-model"}, "weather-deep": {ProfileID: "weather-deep", BackendID: "cloud", ModelName: "deep-model"}, }, } profileIDs := []string{"weather-light", "weather-deep"} result, err := InspectComparisonExecution(context.Background(), ComparisonInspectionRequest{ Resolved: resolved, ProfileIDs: profileIDs, Executor: executor, }) if err != nil { t.Fatalf("InspectComparisonExecution() error = %v", err) } if result.PromptID != resolved.Definition.PromptID || result.PromptVersion != resolved.Definition.PromptVersion || result.PromptHash != "prompt-hash" { t.Fatalf("prompt result = %#v", result) } if !reflect.DeepEqual(executor.profileRequests, profileIDs) || len(executor.promptRequests) != 1 || executor.executeRequests != 0 { t.Fatalf("prompt/profile/execute requests = %#v/%#v/%d", executor.promptRequests, executor.profileRequests, executor.executeRequests) } wantProfiles := []ComparisonProfileInspection{ {ProfileID: "weather-light", BackendID: "local", ModelName: "light-model"}, {ProfileID: "weather-deep", BackendID: "cloud", ModelName: "deep-model"}, } if !reflect.DeepEqual(result.Profiles, wantProfiles) { t.Fatalf("profiles = %#v, want %#v", result.Profiles, wantProfiles) } } func TestInspectComparisonExecutionRejectsInvalidProfilesBeforeInspection(t *testing.T) { resolved := inspectionResolved(t) for _, profileIDs := range [][]string{ {"weather-light"}, {"weather-light", " \t"}, {"weather-light", "weather-light"}, } { t.Run(strings.Join(profileIDs, ","), func(t *testing.T) { executor := &inspectionExecutor{prompt: validPromptInspection(resolved.Definition)} _, err := InspectComparisonExecution(context.Background(), ComparisonInspectionRequest{ Resolved: resolved, ProfileIDs: profileIDs, Executor: executor, }) if err == nil || promptexec.CategoryOf(err) != promptexec.InvalidRequest { t.Fatalf("error/category = %v/%q, want invalid request", err, promptexec.CategoryOf(err)) } if len(executor.promptRequests) != 0 || len(executor.profileRequests) != 0 || executor.executeRequests != 0 { t.Fatalf("invalid profile selection performed prompt/profile/execution work: %#v/%#v/%d", executor.promptRequests, executor.profileRequests, executor.executeRequests) } }) } } func TestInspectComparisonExecutionStopsAtFirstProfileFailure(t *testing.T) { resolved := inspectionResolved(t) executor := &inspectionExecutor{ prompt: validPromptInspection(resolved.Definition), profiles: map[string]promptexec.ProfileInspection{ "weather-light": {ProfileID: "weather-light", BackendID: "local", ModelName: "light-model"}, "missing-key": {ProfileID: "missing-key", APIKeyEnv: "PROMPT_API_KEY"}, "weather-deep": {ProfileID: "weather-deep", BackendID: "cloud", ModelName: "deep-model"}, }, } _, err := InspectComparisonExecution(context.Background(), ComparisonInspectionRequest{ Resolved: resolved, ProfileIDs: []string{"weather-light", "missing-key", "weather-deep"}, Executor: executor, LookupEnv: func(string) (string, bool) { return "", false }, }) if err == nil || promptexec.CategoryOf(err) != promptexec.MissingCredential { t.Fatalf("error/category = %v/%q, want missing credential", err, promptexec.CategoryOf(err)) } if !reflect.DeepEqual(executor.profileRequests, []string{"weather-light", "missing-key"}) || len(executor.promptRequests) != 1 || executor.executeRequests != 0 { t.Fatalf("prompt/profile/execute requests = %#v/%#v/%d", executor.promptRequests, executor.profileRequests, executor.executeRequests) } } func TestInspectComparisonExecutionStopsBeforeProfileInspectionWhenPromptFails(t *testing.T) { resolved := inspectionResolved(t) executor := &inspectionExecutor{promptErr: promptexec.NewError(promptexec.PromptNotFound, "prompt is unavailable", nil)} _, err := InspectComparisonExecution(context.Background(), ComparisonInspectionRequest{ Resolved: resolved, ProfileIDs: []string{"weather-light", "weather-deep"}, Executor: executor, }) if err == nil || promptexec.CategoryOf(err) != promptexec.PromptNotFound || !strings.Contains(err.Error(), "comparison prompt") { t.Fatalf("error/category = %v/%q, want prompt-context prompt not found", err, promptexec.CategoryOf(err)) } if len(executor.promptRequests) != 1 || len(executor.profileRequests) != 0 || executor.executeRequests != 0 { t.Fatalf("prompt/profile/execute requests = %#v/%#v/%d", executor.promptRequests, executor.profileRequests, executor.executeRequests) } } type inspectionPromptRequest struct { id string version string } type inspectionExecutor struct { prompt promptexec.PromptInspection prompts map[string]promptexec.PromptInspection profiles map[string]promptexec.ProfileInspection promptErr error promptRequests []inspectionPromptRequest profileRequests []string executeRequests int } func (e *inspectionExecutor) InspectPrompt(_ context.Context, id string, version string) (promptexec.PromptInspection, error) { e.promptRequests = append(e.promptRequests, inspectionPromptRequest{id: id, version: version}) if e.promptErr != nil { return promptexec.PromptInspection{}, e.promptErr } if prompt, ok := e.prompts[id]; ok { return prompt, nil } return e.prompt, nil } func (e *inspectionExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) { e.profileRequests = append(e.profileRequests, id) value, ok := e.profiles[id] if !ok { return promptexec.ProfileInspection{}, errors.New("profile missing") } return value, nil } func (e *inspectionExecutor) Execute(context.Context, promptexec.ExecuteRequest, promptexec.PreparationCallback) (*promptexec.Execution, error) { e.executeRequests++ return nil, errors.New("unexpected execution") } func inspectionResolved(t *testing.T) report.Resolved { t.Helper() resolved, err := report.DefaultRegistry().Resolve(report.Daily, report.ResolveRequest{ Now: time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC), Date: time.Date(2026, 5, 29, 0, 0, 0, 0, time.UTC), Location: time.UTC, }) if err != nil { t.Fatalf("Resolve() error = %v", err) } return resolved } func validPromptInspection(definition report.Definition) promptexec.PromptInspection { return promptexec.PromptInspection{ PromptID: definition.PromptID, PromptVersion: definition.PromptVersion, PromptHash: "prompt-hash", DefaultProfileID: "default-profile", Inputs: []promptexec.InputDefinition{{Name: "data_package", Required: true, ContentType: "application/yaml"}}, Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: definition.GeneratedTextSchemaID + ".generated_text.schema.json"}, } } func logicalPromptInspection(definition report.Definition) promptexec.PromptInspection { inspection := validPromptInspection(definition) if definition.ID == report.Hourly { inspection.DefaultProfileID = "weather-light" } else { inspection.DefaultProfileID = "weather-balanced" } return inspection }