eric/weatherreporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Correct profile test boundaries and fallback coverage

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-01 17:24:21 +00:00
parent 117c5336ba
commit 1250247986
6 changed files with 393 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
internal/app/single_report_workflow_test.go
View File

@@ -51,6 +51,8 @@ type workflowExecutor struct {
beforeProvider func()
preparationDebug *promptexec.PreparationDebug
executionDebug *promptexec.ExecutionDebug
preparation *promptexec.Preparation
execution *promptexec.Execution
}
func (e *workflowExecutor) InspectPrompt(_ context.Context, id, version string) (promptexec.PromptInspection, error) {
@@ -93,6 +95,7 @@ func (e *workflowExecutor) Execute(_ context.Context, req promptexec.ExecuteRequ
RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID, BackendID: profile.BackendID,
ModelName: profile.ModelName, DataPackagePath: req.DataPackagePath, StartedAt: stamp, EndedAt: stamp,
}
e.preparation = &preparation
if err := callback(preparation, e.preparationDebug); err != nil {
return nil, err
}
@@ -110,14 +113,16 @@ func (e *workflowExecutor) Execute(_ context.Context, req promptexec.ExecuteRequ
if validation == "" {
validation = promptexec.ValidationPassed
}
return &promptexec.Execution{
execution := &promptexec.Execution{
RunID: "provider-run", PromptID: req.PromptID, PromptVersion: req.PromptVersion,
PromptHash: "prompt-hash", RenderedPromptHash: "rendered-hash", ProfileID: req.ProfileID,
BackendID: profile.BackendID, ModelName: profile.ModelName, GeneratedHash: "generated-hash",
StartedAt: stamp, EndedAt: stamp, DataPackagePath: req.DataPackagePath, RawOutput: e.raw,
Debug: e.executionDebug,
Validation: promptexec.NewValidation(validation, "json_schema", e.definition.GeneratedTextSchemaID+".generated_text.schema.json", nil),
}, nil
}
e.execution = execution
return execution, nil
}
type workflowNotifier struct {
@@ -251,7 +256,7 @@ func TestGenerateDetailedPreservesSelectedProfileThroughExecution(t *testing.T)
definition: definition, prompt: logicalPromptInspection(definition), profile: test.profile, raw: []byte(test.raw),
}
bundle := workflowBundle(t)
result, err := GenerateDetailed(context.Background(), GenerateRequest{
_, err := GenerateDetailed(context.Background(), GenerateRequest{
Config: cfg, Report: test.kind, Date: workflowTime("2026-05-29T12:00:00-05:00"), Now: workflowTime("2026-05-29T08:30:00-05:00"),
Collector: &workflowCollector{result: &collect.Result{Bundle: &bundle}}, Executor: executor, Notifier: &workflowNotifier{},
})
@@ -261,23 +266,11 @@ func TestGenerateDetailedPreservesSelectedProfileThroughExecution(t *testing.T)
if executor.request.ProfileID != test.profile.ProfileID {
t.Fatalf("execution profile = %q, want %q", executor.request.ProfileID, test.profile.ProfileID)
}
store, err := state.NewFilesystemStore(cfg.Workspace)
if err != nil {
t.Fatalf("NewFilesystemStore() error = %v", err)
if executor.preparation == nil || executor.preparation.ProfileID != test.profile.ProfileID || executor.preparation.BackendID != test.profile.BackendID || executor.preparation.ModelName != test.profile.ModelName {
t.Fatalf("prepared profile = %#v, want %q/%q/%q", executor.preparation, test.profile.ProfileID, test.profile.BackendID, test.profile.ModelName)
}
preparation, err := store.LoadPromptPreparation(context.Background(), result.PreparationPath)
if err != nil || preparation.Preparation == nil || preparation.Preparation.ProfileID != test.profile.ProfileID || preparation.Preparation.BackendID != test.profile.BackendID || preparation.Preparation.ModelName != test.profile.ModelName {
t.Fatalf("preparation/error = %#v/%v", preparation, err)
}
execution, err := store.LoadPromptExecution(context.Background(), result.ExecutionPath)
if err != nil || execution.Provenance == nil || execution.Provenance.ProfileID != test.profile.ProfileID || execution.Provenance.BackendID != test.profile.BackendID || execution.Provenance.ModelName != test.profile.ModelName {
t.Fatalf("execution/error = %#v/%v", execution, err)
}
for _, path := range []string{result.MetadataPath, result.PreparationPath, result.ExecutionPath} {
data, err := os.ReadFile(path)
if err != nil || strings.Contains(string(data), "https://") || strings.Contains(string(data), "api_key") {
t.Fatalf("ordinary artifact %q leaks sensitive profile details or could not be read: %v", path, err)
}
if executor.execution == nil || executor.execution.ProfileID != test.profile.ProfileID || executor.execution.BackendID != test.profile.BackendID || executor.execution.ModelName != test.profile.ModelName {
t.Fatalf("executed profile = %#v, want %q/%q/%q", executor.execution, test.profile.ProfileID, test.profile.BackendID, test.profile.ModelName)
}
})
}