package app import ( "context" "errors" "strings" "testing" "time" "gitea.maximumdirect.net/eric/weatherreporter/internal/config" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" "gitea.maximumdirect.net/eric/weatherreporter/internal/report" ) func TestInspectPromptExecutionSelectsDefaultAndOverrideProfiles(t *testing.T) { resolved := inspectionResolved(t) executor := &inspectionExecutor{ prompt: validPromptInspection(resolved.Definition), profiles: map[string]promptexec.ProfileInspection{ "default-profile": {ProfileID: "default-profile", BackendID: "local", ModelName: "default-model"}, "override-profile": {ProfileID: "override-profile", BackendID: "cloud", ModelName: "override-model"}, }, } defaultResult, err := InspectPromptExecution(context.Background(), PromptInspectionRequest{Resolved: resolved, Executor: executor}) if err != nil { t.Fatalf("InspectPromptExecution(default) error = %v", err) } if defaultResult.ProfileID != "default-profile" || defaultResult.ModelName != "default-model" { t.Fatalf("default result = %#v", defaultResult) } overrideResult, err := InspectPromptExecution(context.Background(), PromptInspectionRequest{ Resolved: resolved, Executor: executor, Promptkit: config.PromptkitConfig{Profile: "override-profile"}, }) if err != nil { t.Fatalf("InspectPromptExecution(override) error = %v", err) } if overrideResult.ProfileID != "override-profile" || overrideResult.ModelName != "override-model" { t.Fatalf("override result = %#v", overrideResult) } if len(executor.promptRequests) != 2 || executor.promptRequests[0].version != resolved.Definition.PromptVersion || executor.profileRequests[0] != "default-profile" || executor.profileRequests[1] != "override-profile" { t.Fatalf("inspection requests = prompts %#v profiles %#v", executor.promptRequests, executor.profileRequests) } } func TestInspectPromptExecutionRejectsInvalidContractsAndCredentials(t *testing.T) { resolved := inspectionResolved(t) basePrompt := validPromptInspection(resolved.Definition) tests := []struct { name string prompt promptexec.PromptInspection profile promptexec.ProfileInspection lookupEnv func(string) (string, bool) wantCategory promptexec.ErrorCategory }{ { name: "extra input", prompt: func() promptexec.PromptInspection { value := basePrompt value.Inputs = append(value.Inputs, promptexec.InputDefinition{Name: "unexpected"}) return value }(), wantCategory: promptexec.InvalidConfiguration, }, { name: "wrong schema", prompt: func() promptexec.PromptInspection { value := basePrompt value.Output.SchemaPath = "unexpected.schema.json" return value }(), wantCategory: promptexec.InvalidConfiguration, }, { name: "direct key", prompt: basePrompt, profile: promptexec.ProfileInspection{ProfileID: "default-profile", CredentialRequired: true}, wantCategory: promptexec.MissingCredential, }, { name: "missing environment credential", prompt: basePrompt, profile: promptexec.ProfileInspection{ProfileID: "default-profile", APIKeyEnv: "PROMPT_API_KEY"}, lookupEnv: func(string) (string, bool) { return "", false }, wantCategory: promptexec.MissingCredential, }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { executor := &inspectionExecutor{prompt: test.prompt, profiles: map[string]promptexec.ProfileInspection{"default-profile": test.profile}} _, err := InspectPromptExecution(context.Background(), PromptInspectionRequest{Resolved: resolved, Executor: executor, LookupEnv: test.lookupEnv}) if err == nil || promptexec.CategoryOf(err) != test.wantCategory { t.Fatalf("error/category = %v/%q, want %q", err, promptexec.CategoryOf(err), test.wantCategory) } }) } } func TestInspectPromptExecutionReturnsSafeInspectionError(t *testing.T) { resolved := inspectionResolved(t) executor := &inspectionExecutor{promptErr: errors.New("provider response contains resolved-secret-value")} _, err := InspectPromptExecution(context.Background(), PromptInspectionRequest{Resolved: resolved, Executor: executor}) if err == nil || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration { t.Fatalf("error/category = %v/%q", err, promptexec.CategoryOf(err)) } if strings.Contains(err.Error(), "resolved-secret-value") { t.Fatalf("inspection error leaks provider value: %v", err) } } func TestInspectPromptExecutionsReusesEffectiveProfile(t *testing.T) { first := inspectionResolved(t) second := first second.Definition.ID = report.Today second.Definition.PromptID = "weather.today" executor := &inspectionExecutor{ prompt: validPromptInspection(first.Definition), profiles: map[string]promptexec.ProfileInspection{ "default-profile": {ProfileID: "default-profile", BackendID: "local", ModelName: "model"}, }, } executor.prompts = map[string]promptexec.PromptInspection{ first.Definition.PromptID: validPromptInspection(first.Definition), second.Definition.PromptID: validPromptInspection(second.Definition), } results, err := InspectPromptExecutions(context.Background(), PromptExecutionsInspectionRequest{Resolved: []report.Resolved{first, second}, Executor: executor}) if err != nil { t.Fatalf("InspectPromptExecutions() error = %v", err) } if len(results) != 2 || len(executor.profileRequests) != 1 { t.Fatalf("results/profile requests = %#v/%#v, want two results and one profile inspection", results, executor.profileRequests) } } type inspectionPromptRequest struct { id string version string } type inspectionExecutor struct { prompt promptexec.PromptInspection prompts map[string]promptexec.PromptInspection profiles map[string]promptexec.ProfileInspection promptErr error promptRequests []inspectionPromptRequest profileRequests []string } func (e *inspectionExecutor) InspectPrompt(_ context.Context, id string, version string) (promptexec.PromptInspection, error) { e.promptRequests = append(e.promptRequests, inspectionPromptRequest{id: id, version: version}) if e.promptErr != nil { return promptexec.PromptInspection{}, e.promptErr } if prompt, ok := e.prompts[id]; ok { return prompt, nil } return e.prompt, nil } func (e *inspectionExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) { e.profileRequests = append(e.profileRequests, id) value, ok := e.profiles[id] if !ok { return promptexec.ProfileInspection{}, errors.New("profile missing") } return value, nil } func (e *inspectionExecutor) Execute(context.Context, promptexec.ExecuteRequest, promptexec.PreparationCallback) (*promptexec.Execution, error) { return nil, errors.New("unexpected execution") } func inspectionResolved(t *testing.T) report.Resolved { t.Helper() resolved, err := report.DefaultRegistry().Resolve(report.Daily, report.ResolveRequest{ Now: time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC), Date: time.Date(2026, 5, 29, 0, 0, 0, 0, time.UTC), Location: time.UTC, }) if err != nil { t.Fatalf("Resolve() error = %v", err) } return resolved } func validPromptInspection(definition report.Definition) promptexec.PromptInspection { return promptexec.PromptInspection{ PromptID: definition.PromptID, PromptVersion: definition.PromptVersion, PromptHash: "prompt-hash", DefaultProfileID: "default-profile", Inputs: []promptexec.InputDefinition{{Name: "data_package", Required: true, ContentType: "application/yaml"}}, Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: definition.GeneratedTextSchemaID + ".generated_text.schema.json"}, } }