eric/weatherreporter
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Validate generated text catalog before collection

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-13 02:22:09 +00:00
parent a18d5134c7
commit ef2634c2cb
10 changed files with 120 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files

79
internal/app/prompt_inspection_test.go
View File

@@ -110,9 +110,7 @@ func TestInspectPromptExecutionReturnsSafeInspectionError(t *testing.T) {
func TestInspectPromptExecutionsReusesEffectiveProfile(t *testing.T) {
first := inspectionResolved(t)
second := first
second.Definition.ID = report.Today
second.Definition.PromptID = "weather.today"
second := inspectionResolvedFor(t, report.Today)
executor := &inspectionExecutor{
prompt: validPromptInspection(first.Definition),
profiles: map[string]promptexec.ProfileInspection{
@@ -132,6 +130,67 @@ func TestInspectPromptExecutionsReusesEffectiveProfile(t *testing.T) {
}
}
func TestPromptInspectionRejectsIncompatibleGeneratedTextCatalogBeforeExecutorWork(t *testing.T) {
base := inspectionResolved(t)
tests := []struct {
name string
resolved report.Resolved
inspect func(context.Context, report.Resolved, *inspectionExecutor) error
}{
{
name: "single report unknown template",
resolved: func() report.Resolved {
resolved := base
resolved.Definition.TemplateID = "unknown"
return resolved
}(),
inspect: func(ctx context.Context, resolved report.Resolved, executor *inspectionExecutor) error {
_, err := InspectPromptExecution(ctx, PromptInspectionRequest{Resolved: resolved, Executor: executor})
return err
},
},
{
name: "batch known pair for another report",
resolved: func() report.Resolved {
resolved := base
resolved.Definition.GeneratedTextSchemaID = "today"
resolved.Definition.TemplateID = "today"
return resolved
}(),
inspect: func(ctx context.Context, resolved report.Resolved, executor *inspectionExecutor) error {
_, err := InspectPromptExecutions(ctx, PromptExecutionsInspectionRequest{Resolved: []report.Resolved{resolved}, Executor: executor})
return err
},
},
{
name: "comparison known pair for another report",
resolved: func() report.Resolved {
resolved := base
resolved.Definition.GeneratedTextSchemaID = "today"
resolved.Definition.TemplateID = "today"
return resolved
}(),
inspect: func(ctx context.Context, resolved report.Resolved, executor *inspectionExecutor) error {
_, err := InspectComparisonExecution(ctx, ComparisonInspectionRequest{Resolved: resolved, ProfileIDs: []string{"weather-light", "weather-deep"}, Executor: executor})
return err
},
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
executor := &inspectionExecutor{}
err := test.inspect(context.Background(), test.resolved, executor)
if err == nil || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration {
t.Fatalf("inspection error/category = %v/%q, want invalid configuration", err, promptexec.CategoryOf(err))
}
if len(executor.promptRequests) != 0 || len(executor.profileRequests) != 0 || executor.executeRequests != 0 {
t.Fatalf("incompatible catalog performed executor work: prompts %#v profiles %#v executions %d", executor.promptRequests, executor.profileRequests, executor.executeRequests)
}
})
}
}
func TestInspectComparisonExecutionPreservesOrderedExplicitProfiles(t *testing.T) {
resolved := inspectionResolved(t)
executor := &inspectionExecutor{
@@ -265,12 +324,16 @@ func (e *inspectionExecutor) Execute(context.Context, promptexec.ExecuteRequest,
}
func inspectionResolved(t *testing.T) report.Resolved {
return inspectionResolvedFor(t, report.Daily)
}
func inspectionResolvedFor(t *testing.T, id report.ID) report.Resolved {
t.Helper()
resolved, err := report.DefaultRegistry().Resolve(report.Daily, report.ResolveRequest{
Now: time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC),
Date: time.Date(2026, 5, 29, 0, 0, 0, 0, time.UTC),
Location: time.UTC,
})
request := report.ResolveRequest{Now: time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC), Location: time.UTC}
if id == report.Daily {
request.Date = time.Date(2026, 5, 29, 0, 0, 0, 0, time.UTC)
}
resolved, err := report.DefaultRegistry().Resolve(id, request)
if err != nil {
t.Fatalf("Resolve() error = %v", err)
}