eric/weatherreporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Verify profile selection in application workflows

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-01 14:20:57 +00:00
parent 993120a9f2
commit 39c097a710
4 changed files with 102 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
internal/app/batch_execution_test.go
View File

@@ -33,10 +33,10 @@ func TestRunBatchDetailedInspectsEveryCandidateBeforeCollection(t *testing.T) {
t.Fatalf("batchInspectionCandidates() error = %v", err)
}
executor := &inspectionExecutor{profiles: map[string]promptexec.ProfileInspection{
"default-profile": {ProfileID: "default-profile", BackendID: "local", ModelName: "model"},
"weather-balanced": {ProfileID: "weather-balanced", BackendID: "openrouter", ModelName: "~google/gemini-flash-latest"},
}, prompts: map[string]promptexec.PromptInspection{}}
for _, candidate := range candidates {
executor.prompts[candidate.Definition.PromptID] = validPromptInspection(candidate.Definition)
executor.prompts[candidate.Definition.PromptID] = logicalPromptInspection(candidate.Definition)
}
collector := collectorFunc(func(context.Context, collect.Request) (*collect.Result, error) {
return nil, errors.New("collection reached")
@@ -47,7 +47,7 @@ func TestRunBatchDetailedInspectsEveryCandidateBeforeCollection(t *testing.T) {
if err == nil || err.Error() != "collection reached" {
t.Fatalf("RunBatchDetailed() error = %v, want collection error", err)
}
if len(executor.promptRequests) != test.wantPrompts || len(executor.profileRequests) != 1 {
if len(executor.promptRequests) != test.wantPrompts || len(executor.profileRequests) != 1 || executor.profileRequests[0] != "weather-balanced" {
t.Fatalf("inspection calls = prompts %#v profiles %#v", executor.promptRequests, executor.profileRequests)
}
})

6
internal/app/batch_workflow_test.go
View File

@@ -44,7 +44,7 @@ func (e *assembledBatchExecutor) InspectPrompt(_ context.Context, id, version st
if !ok || definition.PromptVersion != version {
return promptexec.PromptInspection{}, errors.New("unexpected prompt inspection")
}
return validPromptInspection(definition), nil
return logicalPromptInspection(definition), nil
}
func (e *assembledBatchExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
@@ -151,8 +151,8 @@ func TestRunBatchDetailedExecutesRetainedReportsSequentially(t *testing.T) {
if len(executor.executeRequests) != len(test.wantIDs) || executor.maxActive != 1 {
t.Fatalf("executor calls/max active = %d/%d, want %d/1", len(executor.executeRequests), executor.maxActive, len(test.wantIDs))
}
if len(executor.profileRequests) != 1 {
t.Fatalf("profile inspections = %#v, want one shared profile inspection", executor.profileRequests)
if len(executor.profileRequests) != 1 || executor.profileRequests[0] != "weather-balanced" {
t.Fatalf("profile inspections = %#v, want one shared weather-balanced inspection", executor.profileRequests)
}
for index, item := range result.Reports {
if item.ReportID != test.wantIDs[index] || item.Status != "succeeded" {

10
internal/app/prompt_inspection_test.go
View File

@@ -189,3 +189,13 @@ func validPromptInspection(definition report.Definition) promptexec.PromptInspec
Output: promptexec.OutputContract{Format: "json", ValidationMode: "json_schema", SchemaPath: definition.GeneratedTextSchemaID + ".generated_text.schema.json"},
}
}
func logicalPromptInspection(definition report.Definition) promptexec.PromptInspection {
inspection := validPromptInspection(definition)
if definition.ID == report.Hourly {
inspection.DefaultProfileID = "weather-light"
} else {
inspection.DefaultProfileID = "weather-balanced"
}
return inspection
}

89
internal/app/single_report_workflow_test.go
View File

@@ -37,8 +37,10 @@ func (c *workflowCollector) Run(context.Context, collect.Request) (*collect.Resu
type workflowExecutor struct {
definition report.Definition
raw []byte
prompt promptexec.PromptInspection
inspectionErr error
profile promptexec.ProfileInspection
profileErr error
beforePreparationErr error
afterCallbackErr error
afterPreparationErr error
@@ -58,10 +60,16 @@ func (e *workflowExecutor) InspectPrompt(_ context.Context, id, version string)
if id != e.definition.PromptID || version != e.definition.PromptVersion {
return promptexec.PromptInspection{}, errors.New("unexpected prompt identity")
}
if e.prompt.PromptID != "" {
return e.prompt, nil
}
return validPromptInspection(e.definition), nil
}
func (e *workflowExecutor) InspectProfile(_ context.Context, id string) (promptexec.ProfileInspection, error) {
if e.profileErr != nil {
return promptexec.ProfileInspection{}, e.profileErr
}
profile := e.profile
if profile.ProfileID == "" {
profile = promptexec.ProfileInspection{ProfileID: id, BackendID: "fixture", ModelName: "fixture-model"}
@@ -76,10 +84,14 @@ func (e *workflowExecutor) Execute(_ context.Context, req promptexec.ExecuteRequ
return nil, e.beforePreparationErr
}
stamp := time.Date(2026, 5, 29, 15, 0, 0, 0, time.UTC)
profile := e.profile
if profile.ProfileID == "" {
profile = promptexec.ProfileInspection{ProfileID: req.ProfileID, BackendID: "fixture", ModelName: "fixture-model"}
}
preparation := promptexec.Preparation{
PromptID: req.PromptID, PromptVersion: req.PromptVersion, PromptHash: "prompt-hash",
RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: "fixture",
ModelName: "fixture-model", DataPackagePath: req.DataPackagePath, StartedAt: stamp, EndedAt: stamp,
RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: profile.BackendID,
ModelName: profile.ModelName, DataPackagePath: req.DataPackagePath, StartedAt: stamp, EndedAt: stamp,
}
if err := callback(preparation, e.preparationDebug); err != nil {
return nil, err
@@ -101,7 +113,7 @@ func (e *workflowExecutor) Execute(_ context.Context, req promptexec.ExecuteRequ
return &promptexec.Execution{
RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion,
PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID,
BackendID: "fixture", ModelName: "fixture-model", GeneratedHash: "generated-hash",
BackendID: profile.BackendID, ModelName: profile.ModelName, GeneratedHash: "generated-hash",
StartedAt: stamp, EndedAt: stamp, DataPackagePath: req.DataPackagePath, RawOutput: e.raw,
Debug: e.executionDebug,
Validation: promptexec.NewValidation(validation, "json_schema", e.definition.GeneratedTextSchemaID+".generated_text.schema.json", nil),
@@ -208,6 +220,69 @@ func TestGenerateDetailedCompletesRetainedReportWorkflows(t *testing.T) {
}
}
func TestGenerateDetailedPreservesSelectedProfileThroughExecution(t *testing.T) {
tests := []struct {
name string
kind ReportKind
id report.ID
raw string
override string
profile promptexec.ProfileInspection
}{
{
name: "hourly default", kind: ReportHourly, id: report.Hourly, raw: validHourlyWorkflowJSON(),
profile: promptexec.ProfileInspection{ProfileID: "weather-light", BackendID: "openrouter", ModelName: "deepseek/deepseek-v4-flash"},
},
{
name: "daily default", kind: ReportDaily, id: report.Daily, raw: validDailyWorkflowJSON(),
profile: promptexec.ProfileInspection{ProfileID: "weather-balanced", BackendID: "openrouter", ModelName: "~google/gemini-flash-latest"},
},
{
name: "global override", kind: ReportDaily, id: report.Daily, raw: validDailyWorkflowJSON(), override: "operator-profile",
profile: promptexec.ProfileInspection{ProfileID: "operator-profile", BackendID: "local", ModelName: "local-weather-model"},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := workflowConfig(t)
cfg.Promptkit.Profile = test.override
definition := report.DefaultRegistry().MustLookup(test.id)
executor := &workflowExecutor{
definition: definition, prompt: logicalPromptInspection(definition), profile: test.profile, raw: []byte(test.raw),
}
bundle := workflowBundle(t)
result, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: test.kind, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"),
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: executor, Notifier: &workflowNotifier{},
})
if err != nil {
t.Fatalf("GenerateDetailed() error = %v", err)
}
if executor.request.ProfileID != test.profile.ProfileID {
t.Fatalf("execution profile = %q, want %q", executor.request.ProfileID, test.profile.ProfileID)
}
store, err := state.NewFilesystemStore(cfg.Workspace)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
}
preparation, err := store.LoadPromptPreparation(context.Background(), result.PreparationPath)
if err != nil || preparation.Preparation == nil || preparation.Preparation.ProfileID != test.profile.ProfileID || preparation.Preparation.BackendID != test.profile.BackendID || preparation.Preparation.ModelName != test.profile.ModelName {
t.Fatalf("preparation/error = %#v/%v", preparation, err)
}
execution, err := store.LoadPromptExecution(context.Background(), result.ExecutionPath)
if err != nil || execution.Provenance == nil || execution.Provenance.ProfileID != test.profile.ProfileID || execution.Provenance.BackendID != test.profile.BackendID || execution.Provenance.ModelName != test.profile.ModelName {
t.Fatalf("execution/error = %#v/%v", execution, err)
}
for _, path := range []string{result.MetadataPath, result.PreparationPath, result.ExecutionPath} {
data, err := os.ReadFile(path)
if err != nil || strings.Contains(string(data), "https://") || strings.Contains(string(data), "api_key") {
t.Fatalf("ordinary artifact %q leaks sensitive profile details or could not be read: %v", path, err)
}
}
})
}
}
type preparationFailingStore struct {
state.Store
}
@@ -299,6 +374,11 @@ func TestGenerateDetailedRejectsInspectionAndCredentialsBeforeCollection(t *test
wantCategory promptexec.ErrorCategory
}{
{name: "inspection", configure: func(e *workflowExecutor) { e.inspectionErr = errors.New("inspection unavailable") }, wantCategory: promptexec.InvalidConfiguration},
{name: "unknown profile", configure: func(e *workflowExecutor) { e.profileErr = errors.New("unknown selected profile") }, wantCategory: promptexec.InvalidConfiguration},
{name: "malformed profile", configure: func(e *workflowExecutor) {
e.profileErr = errors.New("malformed profile at https://operator.example/v1 api_key=secret")
}, wantCategory: promptexec.InvalidConfiguration},
{name: "unusable backend", configure: func(e *workflowExecutor) { e.profileErr = errors.New("unsupported backend") }, wantCategory: promptexec.InvalidConfiguration},
{name: "credential", configure: func(e *workflowExecutor) {
e.profile = promptexec.ProfileInspection{ProfileID: "default-profile", CredentialRequired: true}
}, wantCategory: promptexec.MissingCredential},
@@ -317,6 +397,9 @@ func TestGenerateDetailedRejectsInspectionAndCredentialsBeforeCollection(t *test
if err == nil || result != nil || promptexec.CategoryOf(err) != test.wantCategory || collector.calls != 0 || executor.executeCalls != 0 {
t.Fatalf("result/error/category/collect/execute = %#v/%v/%q/%d/%d", result, err, promptexec.CategoryOf(err), collector.calls, executor.executeCalls)
}
if strings.Contains(err.Error(), "operator.example") || strings.Contains(err.Error(), "secret") {
t.Fatalf("error leaks profile details: %v", err)
}
entries, readErr := os.ReadDir(cfg.Workspace.Root)
if readErr != nil || len(entries) != 0 {
t.Fatalf("workspace entries/error = %#v/%v, want no writes before collection", entries, readErr)