From 39c097a7109deb6dc8787d18381313cef1d7aff2 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Sat, 1 Aug 2026 14:20:57 +0000 Subject: [PATCH] Verify profile selection in application workflows --- internal/app/batch_execution_test.go | 6 +- internal/app/batch_workflow_test.go | 6 +- internal/app/prompt_inspection_test.go | 10 +++ internal/app/single_report_workflow_test.go | 89 ++++++++++++++++++++- 4 files changed, 102 insertions(+), 9 deletions(-) diff --git a/internal/app/batch_execution_test.go b/internal/app/batch_execution_test.go index 4e04da6..21afb9e 100644 --- a/internal/app/batch_execution_test.go +++ b/internal/app/batch_execution_test.go @@ -33,10 +33,10 @@ func TestRunBatchDetailedInspectsEveryCandidateBeforeCollection(t *testing.T) { t.Fatalf("batchInspectionCandidates() error = %v", err) } executor := &inspectionExecutor{profiles: map[string]promptexec.ProfileInspection{ - "default-profile": {ProfileID: "default-profile", BackendID: "local", ModelName: "model"}, + "weather-balanced": {ProfileID: "weather-balanced", BackendID: "openrouter", ModelName: "~google/gemini-flash-latest"}, }, prompts: map[string]promptexec.PromptInspection{}} for _, candidate := range candidates { - executor.prompts[candidate.Definition.PromptID] = validPromptInspection(candidate.Definition) + executor.prompts[candidate.Definition.PromptID] = logicalPromptInspection(candidate.Definition) } collector := collectorFunc(func(context.Context, collect.Request) (*collect.Result, error) { return nil, errors.New("collection reached") @@ -47,7 +47,7 @@ func TestRunBatchDetailedInspectsEveryCandidateBeforeCollection(t *testing.T) { if err == nil || err.Error() != "collection reached" { t.Fatalf("RunBatchDetailed() error = %v, want collection error", err) } - if len(executor.promptRequests) != test.wantPrompts || len(executor.profileRequests) != 1 { + if len(executor.promptRequests) != test.wantPrompts || len(executor.profileRequests) != 1 || executor.profileRequests[0] != "weather-balanced" { t.Fatalf("inspection calls = prompts %#v profiles %#v", executor.promptRequests, executor.profileRequests) } }) diff --git a/internal/app/batch_workflow_test.go b/internal/app/batch_workflow_test.go index 7710d2a..85f1073 100644 --- a/internal/app/batch_workflow_test.go +++ b/internal/app/batch_workflow_test.go @@ -44,7 +44,7 @@ func (e *assembledBatchExecutor) InspectPrompt(_ context.Context, id, version st if !ok || definition.PromptVersion != version { return promptexec.PromptInspection{}, errors.New("unexpected prompt inspection") } - return validPromptInspection(definition), nil + return logicalPromptInspection(definition), nil } func (e *assembledBatchExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) { @@ -151,8 +151,8 @@ func TestRunBatchDetailedExecutesRetainedReportsSequentially(t *testing.T) { if len(executor.executeRequests) != len(test.wantIDs) || executor.maxActive != 1 { t.Fatalf("executor calls/max active = %d/%d, want %d/1", len(executor.executeRequests), executor.maxActive, len(test.wantIDs)) } - if len(executor.profileRequests) != 1 { - t.Fatalf("profile inspections = %#v, want one shared profile inspection", executor.profileRequests) + if len(executor.profileRequests) != 1 || executor.profileRequests[0] != "weather-balanced" { + t.Fatalf("profile inspections = %#v, want one shared weather-balanced inspection", executor.profileRequests) } for index, item := range result.Reports { if item.ReportID != test.wantIDs[index] || item.Status != "succeeded" { diff --git a/internal/app/prompt_inspection_test.go b/internal/app/prompt_inspection_test.go index 89d5cdb..15ecbb9 100644 --- a/internal/app/prompt_inspection_test.go +++ b/internal/app/prompt_inspection_test.go @@ -189,3 +189,13 @@ func validPromptInspection(definition report.Definition) promptexec.PromptInspec Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: definition.GeneratedTextSchemaID + ".generated_text.schema.json"}, } } + +func logicalPromptInspection(definition report.Definition) promptexec.PromptInspection { + inspection := validPromptInspection(definition) + if definition.ID == report.Hourly { + inspection.DefaultProfileID = "weather-light" + } else { + inspection.DefaultProfileID = "weather-balanced" + } + return inspection +} diff --git a/internal/app/single_report_workflow_test.go b/internal/app/single_report_workflow_test.go index 9737216..de2789c 100644 --- a/internal/app/single_report_workflow_test.go +++ b/internal/app/single_report_workflow_test.go @@ -37,8 +37,10 @@ func (c *workflowCollector) Run(context.Context, collect.Request) (*collect.Resu type workflowExecutor struct { definition report.Definition raw []byte + prompt promptexec.PromptInspection inspectionErr error profile promptexec.ProfileInspection + profileErr error beforePreparationErr error afterCallbackErr error afterPreparationErr error @@ -58,10 +60,16 @@ func (e *workflowExecutor) InspectPrompt(_ context.Context, id, version string) if id != e.definition.PromptID || version != e.definition.PromptVersion { return promptexec.PromptInspection{}, errors.New("unexpected prompt identity") } + if e.prompt.PromptID != "" { + return e.prompt, nil + } return validPromptInspection(e.definition), nil } func (e *workflowExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) { + if e.profileErr != nil { + return promptexec.ProfileInspection{}, e.profileErr + } profile := e.profile if profile.ProfileID == "" { profile = promptexec.ProfileInspection{ProfileID: id, BackendID: "fixture", ModelName: "fixture-model"} @@ -76,10 +84,14 @@ func (e *workflowExecutor) Execute(_ context.Context, req promptexec.ExecuteRequ return nil, e.beforePreparationErr } stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC) + profile := e.profile + if profile.ProfileID == "" { + profile = promptexec.ProfileInspection{ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model"} + } preparation := promptexec.Preparation{ PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", - RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture", - ModelName: "fixture-model", DataPackagePath: req.DataPackagePath, StartedAt: stamp, EndedAt: stamp, + RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: profile.BackendID, + ModelName: profile.ModelName, DataPackagePath: req.DataPackagePath, StartedAt: stamp, EndedAt: stamp, } if err := callback(preparation, e.preparationDebug); err != nil { return nil, err @@ -101,7 +113,7 @@ func (e *workflowExecutor) Execute(_ context.Context, req promptexec.ExecuteRequ return &promptexec.Execution{ RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, - BackendID: "fixture", ModelName: "fixture-model", GeneratedHash: "generated-hash", + BackendID: profile.BackendID, ModelName: profile.ModelName, GeneratedHash: "generated-hash", StartedAt: stamp, EndedAt: stamp, DataPackagePath: req.DataPackagePath, RawOutput: e.raw, Debug: e.executionDebug, Validation: promptexec.NewValidation(validation, "json_schema", e.definition.GeneratedTextSchemaID+".generated_text.schema.json", nil), @@ -208,6 +220,69 @@ func TestGenerateDetailedCompletesRetainedReportWorkflows(t *testing.T) { } } +func TestGenerateDetailedPreservesSelectedProfileThroughExecution(t *testing.T) { + tests := []struct { + name string + kind ReportKind + id report.ID + raw string + override string + profile promptexec.ProfileInspection + }{ + { + name: "hourly default", kind: ReportHourly, id: report.Hourly, raw: validHourlyWorkflowJSON(), + profile: promptexec.ProfileInspection{ProfileID: "weather-light", BackendID: "openrouter", ModelName: "deepseek/deepseek-v4-flash"}, + }, + { + name: "daily default", kind: ReportDaily, id: report.Daily, raw: validDailyWorkflowJSON(), + profile: promptexec.ProfileInspection{ProfileID: "weather-balanced", BackendID: "openrouter", ModelName: "~google/gemini-flash-latest"}, + }, + { + name: "global override", kind: ReportDaily, id: report.Daily, raw: validDailyWorkflowJSON(), override: "operator-profile", + profile: promptexec.ProfileInspection{ProfileID: "operator-profile", BackendID: "local", ModelName: "local-weather-model"}, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cfg := workflowConfig(t) + cfg.Promptkit.Profile = test.override + definition := report.DefaultRegistry().MustLookup(test.id) + executor := &workflowExecutor{ + definition: definition, prompt: logicalPromptInspection(definition), profile: test.profile, raw: []byte(test.raw), + } + bundle := workflowBundle(t) + result, err := GenerateDetailed(context.Background(), GenerateRequest{ + Config: cfg, Report: test.kind, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"), + Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: executor, Notifier: &workflowNotifier{}, + }) + if err != nil { + t.Fatalf("GenerateDetailed() error = %v", err) + } + if executor.request.ProfileID != test.profile.ProfileID { + t.Fatalf("execution profile = %q, want %q", executor.request.ProfileID, test.profile.ProfileID) + } + store, err := state.NewFilesystemStore(cfg.Workspace) + if err != nil { + t.Fatalf("NewFilesystemStore() error = %v", err) + } + preparation, err := store.LoadPromptPreparation(context.Background(), result.PreparationPath) + if err != nil || preparation.Preparation == nil || preparation.Preparation.ProfileID != test.profile.ProfileID || preparation.Preparation.BackendID != test.profile.BackendID || preparation.Preparation.ModelName != test.profile.ModelName { + t.Fatalf("preparation/error = %#v/%v", preparation, err) + } + execution, err := store.LoadPromptExecution(context.Background(), result.ExecutionPath) + if err != nil || execution.Provenance == nil || execution.Provenance.ProfileID != test.profile.ProfileID || execution.Provenance.BackendID != test.profile.BackendID || execution.Provenance.ModelName != test.profile.ModelName { + t.Fatalf("execution/error = %#v/%v", execution, err) + } + for _, path := range []string{result.MetadataPath, result.PreparationPath, result.ExecutionPath} { + data, err := os.ReadFile(path) + if err != nil || strings.Contains(string(data), "https://") || strings.Contains(string(data), "api_key") { + t.Fatalf("ordinary artifact %q leaks sensitive profile details or could not be read: %v", path, err) + } + } + }) + } +} + type preparationFailingStore struct { state.Store } @@ -299,6 +374,11 @@ func TestGenerateDetailedRejectsInspectionAndCredentialsBeforeCollection(t *test wantCategory promptexec.ErrorCategory }{ {name: "inspection", configure: func(e *workflowExecutor) { e.inspectionErr = errors.New("inspection unavailable") }, wantCategory: promptexec.InvalidConfiguration}, + {name: "unknown profile", configure: func(e *workflowExecutor) { e.profileErr = errors.New("unknown selected profile") }, wantCategory: promptexec.InvalidConfiguration}, + {name: "malformed profile", configure: func(e *workflowExecutor) { + e.profileErr = errors.New("malformed profile at https://operator.example/v1 api_key=secret") + }, wantCategory: promptexec.InvalidConfiguration}, + {name: "unusable backend", configure: func(e *workflowExecutor) { e.profileErr = errors.New("unsupported backend") }, wantCategory: promptexec.InvalidConfiguration}, {name: "credential", configure: func(e *workflowExecutor) { e.profile = promptexec.ProfileInspection{ProfileID: "default-profile", CredentialRequired: true} }, wantCategory: promptexec.MissingCredential}, @@ -317,6 +397,9 @@ func TestGenerateDetailedRejectsInspectionAndCredentialsBeforeCollection(t *test if err == nil || result != nil || promptexec.CategoryOf(err) != test.wantCategory || collector.calls != 0 || executor.executeCalls != 0 { t.Fatalf("result/error/category/collect/execute = %#v/%v/%q/%d/%d", result, err, promptexec.CategoryOf(err), collector.calls, executor.executeCalls) } + if strings.Contains(err.Error(), "operator.example") || strings.Contains(err.Error(), "secret") { + t.Fatalf("error leaks profile details: %v", err) + } entries, readErr := os.ReadDir(cfg.Workspace.Root) if readErr != nil || len(entries) != 0 { t.Fatalf("workspace entries/error = %#v/%v, want no writes before collection", entries, readErr)