From ef2634c2cbadd4384d63cf8ce378a319536764be Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Thu, 13 Aug 2026 02:22:09 +0000 Subject: [PATCH] Validate generated text catalog before collection --- docs/integrations/promptkit.md | 7 ++- docs/internal/app-orchestration.md | 9 +-- docs/internal/prepared-report.md | 14 +++-- internal/app/comparison.go | 2 +- internal/app/prepared_report.go | 8 +-- internal/app/prepared_report_test.go | 15 ++++- internal/app/profile_execution_test.go | 2 +- internal/app/prompt_generate.go | 2 +- internal/app/prompt_inspection.go | 13 +++++ internal/app/prompt_inspection_test.go | 79 +++++++++++++++++++++++--- 10 files changed, 120 insertions(+), 31 deletions(-) diff --git a/docs/integrations/promptkit.md b/docs/integrations/promptkit.md index b73fea7..b72e1d7 100644 --- a/docs/integrations/promptkit.md +++ b/docs/integrations/promptkit.md @@ -16,7 +16,7 @@ The `~` prefix is part of each OpenRouter rolling-alias model ID. The embedded p ## Selection And Active Execution -Before weather collection, Weatherreporter validates the exact prompt version, output contract, and selected profile. A nonblank `promptkit.profile` selects one profile ID for every report in the command; otherwise the prompt's declared default selects it. Promptkit resolves the selected definition in this order: +Before weather collection, Weatherreporter validates the report's exact generated-text report/schema/template catalog binding, prompt version, output contract, and selected profile. A nonblank `promptkit.profile` selects one profile ID for every report in the command; otherwise the prompt's declared default selects it. Promptkit resolves the selected definition in this order: 1. explicit in-memory profiles used by an embedding consumer or test; 2. the configured `profile_file` or `profile_dir`; @@ -37,8 +37,9 @@ filesystem path causes the requested execution to fail. ## Comparison Execution -For `compare`, Weatherreporter validates one exact prompt and every explicitly -selected profile before weather collection. It prepares one deterministic YAML +For `compare`, Weatherreporter validates the report's generated-text catalog +binding, one exact prompt, and every explicitly selected profile before weather +collection. It prepares one deterministic YAML data package, retains immutable copies of the report inputs, and executes every profile against the same exact data-package bytes. Each profile remains an independent Promptkit execution: one provider or validation failure does not diff --git a/docs/internal/app-orchestration.md b/docs/internal/app-orchestration.md index 88ddf34..e9d0c62 100644 --- a/docs/internal/app-orchestration.md +++ b/docs/internal/app-orchestration.md @@ -7,7 +7,7 @@ is owned by the [CLI reference](../cli.md) and [operations guide](../operations. ## Single-Report Flow -`GenerateDetailed` resolves the requested report and output destination before initializing an optional explicit debug writer. An explicit output file wins; otherwise the configured output directory is used, falling back to the captured working directory. It validates the exact Promptkit prompt and selected profile before collecting weather data. The resolved profile, backend, and model are carried in the active result. +`GenerateDetailed` resolves the requested report and output destination before initializing an optional explicit debug writer. An explicit output file wins; otherwise the configured output directory is used, falling back to the captured working directory. It validates the report's generated-text catalog binding, exact Promptkit prompt, and selected profile before collecting weather data. The resolved profile, backend, and model are carried in the active result. The workflow builds facts, a module snapshot, briefing metadata, and the YAML prompt package in memory. It executes Promptkit, validates the returned generated text, builds a render context, and renders Markdown. `fileutil` atomically writes the completed Markdown to the selected output path. Only after that write succeeds does single-report notification run. @@ -15,7 +15,7 @@ Failures return an active partial result with safe identity, profile, warning, v ## Batches -`RunBatchDetailed` selects an explicit output directory first, otherwise the configured directory and then the captured working directory. It does this before creating at most one explicit debug writer or validating prompt and profile candidates for the selected batch. It collects once, calculates the data-dependent plan, then validates and retains the final output path for every planned report before invoking the same generation core sequentially. +`RunBatchDetailed` selects an explicit output directory first, otherwise the configured directory and then the captured working directory. It does this before creating at most one explicit debug writer or validating generated-text catalog, prompt, and profile candidates for the selected batch. It collects once, calculates the data-dependent plan, then validates and retains the final output path for every planned report before invoking the same generation core sequentially. Each item has an independent result. A failed item does not stop later items; successful items retain their published output paths. Per-report notification is suppressed during a batch. Batch notification runs only after every planned report has published successfully. It is skipped when any item failed. Batch result counters count report items only; a batch notification failure is represented by the top-level notification result and still produces a failed batch outcome. @@ -23,8 +23,9 @@ Each item has an independent result. A failed item does not stop later items; su `CompareDetailed` validates ordered explicit profile IDs, resolves the report, and preflights the exact bundle destination before initializing optional prompt -debugging, prompt inspection, or collection. It then inspects the one prompt -and every selected profile, collects once, and delegates shared report +debugging, prompt inspection, or collection. It then validates the report's +generated-text catalog binding, inspects the one prompt and every selected +profile, collects once, and delegates shared report construction to the prepared-report flow. It does not accept a notifier. Once the destination is resolved, the partial result retains its absolute diff --git a/docs/internal/prepared-report.md b/docs/internal/prepared-report.md index b0c9a0b..0e49d94 100644 --- a/docs/internal/prepared-report.md +++ b/docs/internal/prepared-report.md @@ -1,8 +1,10 @@ # Prepared Report Internals -`internal/app` builds a `preparedReport` after collection and before profile -execution. This is the immutable boundary shared by ordinary report generation -and profile comparison; it is not a durable artifact. +`internal/app` validates the report's generated-text catalog binding during +prompt inspection, before collection, and carries the resulting handler into +`preparedReport` construction after collection. This is the immutable boundary +shared by ordinary report generation and profile comparison; it is not a +durable artifact. Preparation first establishes one `PreparedIdentity` for the report run, report and prompt IDs, variant, generation time, units, timezone, valid period, @@ -21,5 +23,7 @@ bytes, and only then assembles the resulting logical bundle. The prompt-input shape is owned by [prompt-input internals](prompt-input.md); profile execution semantics are owned by [Promptkit integration](../integrations/promptkit.md). -Preparation failure has no publication side effects. Tests for this boundary -cover mutation isolation, byte equality, and reuse by both execution paths. +Catalog incompatibility stops prompt inspection before weather collection or +model work. Preparation failure has no publication side effects. Tests for this +boundary cover catalog-preflight ordering, mutation isolation, byte equality, +and reuse by both execution paths. diff --git a/internal/app/comparison.go b/internal/app/comparison.go index eacaf2c..eea496a 100644 --- a/internal/app/comparison.go +++ b/internal/app/comparison.go @@ -132,7 +132,7 @@ func compareDetailed(ctx context.Context, req ComparisonRequest, publish compari if err != nil { return result, err } - prepared, err := prepareReport(prepareReportRequest{Config: req.Config, Resolved: resolved, Collection: *collection}) + prepared, err := prepareReport(prepareReportRequest{Config: req.Config, Resolved: resolved, Collection: *collection, handler: inspection.handler}) if err != nil { return result, fmt.Errorf("prepare comparison report: %w", err) } diff --git a/internal/app/prepared_report.go b/internal/app/prepared_report.go index e1da3f8..0d6fb35 100644 --- a/internal/app/prepared_report.go +++ b/internal/app/prepared_report.go @@ -31,6 +31,7 @@ type prepareReportRequest struct { Config config.Config Resolved report.Resolved Collection collect.Result + handler generatedtext.Handler } type preparationError struct { @@ -69,11 +70,6 @@ func prepareReport(req prepareReportRequest) (preparedReport, error) { if err != nil { return preparedReport{}, &preparationError{operation: "marshal data package", err: err} } - handler, err := generatedtext.LookupDefinition(req.Resolved.Definition) - if err != nil { - return preparedReport{}, &preparationError{operation: "lookup generated text catalog", err: err} - } - clonedDerived, err := clonePreparedValue(reportFacts.Derived) if err != nil { return preparedReport{}, &preparationError{operation: "copy prepared derived facts", err: err} @@ -93,7 +89,7 @@ func prepareReport(req prepareReportRequest) (preparedReport, error) { identity: clonedIdentity, sourceWarnings: append([]weatherdata.SourceWarning(nil), clonedIdentity.SourceWarnings...), dataPackage: append([]byte(nil), serializedDataPackage...), - handler: handler, + handler: req.handler, } return prepared, nil } diff --git a/internal/app/prepared_report_test.go b/internal/app/prepared_report_test.go index 000add7..9360d79 100644 --- a/internal/app/prepared_report_test.go +++ b/internal/app/prepared_report_test.go @@ -7,7 +7,9 @@ import ( "gitea.maximumdirect.net/eric/weatherreporter/internal/briefing" "gitea.maximumdirect.net/eric/weatherreporter/internal/collect" + "gitea.maximumdirect.net/eric/weatherreporter/internal/generatedtext" "gitea.maximumdirect.net/eric/weatherreporter/internal/module" + "gitea.maximumdirect.net/eric/weatherreporter/internal/report" "gitea.maximumdirect.net/eric/weatherreporter/internal/weatherdata" ) @@ -22,7 +24,7 @@ func TestPrepareReportBuildsImmutableDeterministicInputs(t *testing.T) { t.Fatalf("ResolveGenerate() error = %v", err) } - request := prepareReportRequest{Config: cfg, Resolved: resolved, Collection: collect.Result{Bundle: &bundle}} + request := prepareReportRequest{Config: cfg, Resolved: resolved, Collection: collect.Result{Bundle: &bundle}, handler: preparedHandler(t, resolved)} prepared, err := prepareReport(request) if err != nil { t.Fatalf("prepareReport() error = %v", err) @@ -73,7 +75,7 @@ func TestPrepareReportProjectsPreparedIdentity(t *testing.T) { if err != nil { t.Fatalf("ResolveGenerate() error = %v", err) } - prepared, err := prepareReport(prepareReportRequest{Config: cfg, Resolved: resolved, Collection: collect.Result{Bundle: &bundle}}) + prepared, err := prepareReport(prepareReportRequest{Config: cfg, Resolved: resolved, Collection: collect.Result{Bundle: &bundle}, handler: preparedHandler(t, resolved)}) if err != nil { t.Fatalf("prepareReport() error = %v", err) } @@ -108,3 +110,12 @@ func TestPrepareReportProjectsPreparedIdentity(t *testing.T) { } } } + +func preparedHandler(t *testing.T, resolved report.Resolved) generatedtext.Handler { + t.Helper() + handler, err := generatedtext.LookupDefinition(resolved.Definition) + if err != nil { + t.Fatalf("LookupDefinition() error = %v", err) + } + return handler +} diff --git a/internal/app/profile_execution_test.go b/internal/app/profile_execution_test.go index 68e10f9..1b4d17f 100644 --- a/internal/app/profile_execution_test.go +++ b/internal/app/profile_execution_test.go @@ -81,7 +81,7 @@ func preparedDailyProfile(t *testing.T) (preparedReport, PromptInspectionResult) t.Fatalf("ResolveGenerate() error = %v", err) } bundle := generationBundle(t) - prepared, err := prepareReport(prepareReportRequest{Config: cfg, Resolved: resolved, Collection: collect.Result{Bundle: &bundle}}) + prepared, err := prepareReport(prepareReportRequest{Config: cfg, Resolved: resolved, Collection: collect.Result{Bundle: &bundle}, handler: preparedHandler(t, resolved)}) if err != nil { t.Fatalf("prepareReport() error = %v", err) } diff --git a/internal/app/prompt_generate.go b/internal/app/prompt_generate.go index ab04498..b7e5caa 100644 --- a/internal/app/prompt_generate.go +++ b/internal/app/prompt_generate.go @@ -30,7 +30,7 @@ func generatePromptReport(ctx context.Context, req promptReportRequest) (*Report if result == nil { result = initialReportResult(req.GenerateRequest, req.Resolved, req.Inspection) } - prepared, err := prepareReport(prepareReportRequest{Config: req.Config, Resolved: req.Resolved, Collection: req.Collection}) + prepared, err := prepareReport(prepareReportRequest{Config: req.Config, Resolved: req.Resolved, Collection: req.Collection, handler: req.Inspection.handler}) if err != nil { return result, generatedPreparationError(req.Resolved, result.RunID, err) } diff --git a/internal/app/prompt_inspection.go b/internal/app/prompt_inspection.go index bf8b67a..7d7b271 100644 --- a/internal/app/prompt_inspection.go +++ b/internal/app/prompt_inspection.go @@ -7,6 +7,7 @@ import ( "gitea.maximumdirect.net/eric/weatherreporter/internal/comparison" "gitea.maximumdirect.net/eric/weatherreporter/internal/config" + "gitea.maximumdirect.net/eric/weatherreporter/internal/generatedtext" "gitea.maximumdirect.net/eric/weatherreporter/internal/promptexec" "gitea.maximumdirect.net/eric/weatherreporter/internal/report" ) @@ -29,6 +30,7 @@ type PromptInspectionResult struct { ProfileID string BackendID string ModelName string + handler generatedtext.Handler } // PromptExecutionsInspectionRequest validates all prompt/profile combinations @@ -56,6 +58,7 @@ type ComparisonInspectionResult struct { PromptVersion string PromptHash string Profiles []ComparisonProfileInspection + handler generatedtext.Handler } // ComparisonProfileInspection contains one requested profile's safe effective @@ -91,6 +94,10 @@ func InspectPromptExecutions(ctx context.Context, req PromptExecutionsInspection profiles := map[string]promptexec.ProfileInspection{} for _, resolved := range req.Resolved { definition := resolved.Definition + handler, err := generatedtext.LookupDefinition(definition) + if err != nil { + return nil, promptexec.NewError(promptexec.InvalidConfiguration, "report generated-text catalog is incompatible", err) + } inspection, err := inspectPromptContract(ctx, req.Executor, definition) if err != nil { return nil, err @@ -113,6 +120,7 @@ func InspectPromptExecutions(ctx context.Context, req PromptExecutionsInspection results[definition.ID] = PromptInspectionResult{ PromptID: inspection.PromptID, PromptVersion: inspection.PromptVersion, PromptHash: inspection.PromptHash, ProfileID: profile.ProfileID, BackendID: profile.BackendID, ModelName: profile.ModelName, + handler: handler, } } return results, nil @@ -130,6 +138,10 @@ func InspectComparisonExecution(ctx context.Context, req ComparisonInspectionReq return ComparisonInspectionResult{}, promptexec.NewError(promptexec.InvalidConfiguration, "prompt executor is required", nil) } + handler, err := generatedtext.LookupDefinition(req.Resolved.Definition) + if err != nil { + return ComparisonInspectionResult{}, comparisonInspectionError("comparison generated-text catalog inspection failed", promptexec.NewError(promptexec.InvalidConfiguration, "report generated-text catalog is incompatible", err)) + } inspection, err := inspectPromptContract(ctx, req.Executor, req.Resolved.Definition) if err != nil { return ComparisonInspectionResult{}, comparisonInspectionError("comparison prompt inspection failed", err) @@ -139,6 +151,7 @@ func InspectComparisonExecution(ctx context.Context, req ComparisonInspectionReq PromptVersion: inspection.PromptVersion, PromptHash: inspection.PromptHash, Profiles: make([]ComparisonProfileInspection, 0, len(req.ProfileIDs)), + handler: handler, } for _, profileID := range req.ProfileIDs { profile, err := inspectPromptProfile(ctx, req.Executor, profileID, req.LookupEnv) diff --git a/internal/app/prompt_inspection_test.go b/internal/app/prompt_inspection_test.go index 5727a1e..9426783 100644 --- a/internal/app/prompt_inspection_test.go +++ b/internal/app/prompt_inspection_test.go @@ -110,9 +110,7 @@ func TestInspectPromptExecutionReturnsSafeInspectionError(t *testing.T) { func TestInspectPromptExecutionsReusesEffectiveProfile(t *testing.T) { first := inspectionResolved(t) - second := first - second.Definition.ID = report.Today - second.Definition.PromptID = "weather.today" + second := inspectionResolvedFor(t, report.Today) executor := &inspectionExecutor{ prompt: validPromptInspection(first.Definition), profiles: map[string]promptexec.ProfileInspection{ @@ -132,6 +130,67 @@ func TestInspectPromptExecutionsReusesEffectiveProfile(t *testing.T) { } } +func TestPromptInspectionRejectsIncompatibleGeneratedTextCatalogBeforeExecutorWork(t *testing.T) { + base := inspectionResolved(t) + tests := []struct { + name string + resolved report.Resolved + inspect func(context.Context, report.Resolved, *inspectionExecutor) error + }{ + { + name: "single report unknown template", + resolved: func() report.Resolved { + resolved := base + resolved.Definition.TemplateID = "unknown" + return resolved + }(), + inspect: func(ctx context.Context, resolved report.Resolved, executor *inspectionExecutor) error { + _, err := InspectPromptExecution(ctx, PromptInspectionRequest{Resolved: resolved, Executor: executor}) + return err + }, + }, + { + name: "batch known pair for another report", + resolved: func() report.Resolved { + resolved := base + resolved.Definition.GeneratedTextSchemaID = "today" + resolved.Definition.TemplateID = "today" + return resolved + }(), + inspect: func(ctx context.Context, resolved report.Resolved, executor *inspectionExecutor) error { + _, err := InspectPromptExecutions(ctx, PromptExecutionsInspectionRequest{Resolved: []report.Resolved{resolved}, Executor: executor}) + return err + }, + }, + { + name: "comparison known pair for another report", + resolved: func() report.Resolved { + resolved := base + resolved.Definition.GeneratedTextSchemaID = "today" + resolved.Definition.TemplateID = "today" + return resolved + }(), + inspect: func(ctx context.Context, resolved report.Resolved, executor *inspectionExecutor) error { + _, err := InspectComparisonExecution(ctx, ComparisonInspectionRequest{Resolved: resolved, ProfileIDs: []string{"weather-light", "weather-deep"}, Executor: executor}) + return err + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + executor := &inspectionExecutor{} + err := test.inspect(context.Background(), test.resolved, executor) + if err == nil || promptexec.CategoryOf(err) != promptexec.InvalidConfiguration { + t.Fatalf("inspection error/category = %v/%q, want invalid configuration", err, promptexec.CategoryOf(err)) + } + if len(executor.promptRequests) != 0 || len(executor.profileRequests) != 0 || executor.executeRequests != 0 { + t.Fatalf("incompatible catalog performed executor work: prompts %#v profiles %#v executions %d", executor.promptRequests, executor.profileRequests, executor.executeRequests) + } + }) + } +} + func TestInspectComparisonExecutionPreservesOrderedExplicitProfiles(t *testing.T) { resolved := inspectionResolved(t) executor := &inspectionExecutor{ @@ -265,12 +324,16 @@ func (e *inspectionExecutor) Execute(context.Context, promptexec.ExecuteRequest, } func inspectionResolved(t *testing.T) report.Resolved { + return inspectionResolvedFor(t, report.Daily) +} + +func inspectionResolvedFor(t *testing.T, id report.ID) report.Resolved { t.Helper() - resolved, err := report.DefaultRegistry().Resolve(report.Daily, report.ResolveRequest{ - Now: time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC), - Date: time.Date(2026, 5, 29, 0, 0, 0, 0, time.UTC), - Location: time.UTC, - }) + request := report.ResolveRequest{Now: time.Date(2026, 5, 29, 12, 0, 0, 0, time.UTC), Location: time.UTC} + if id == report.Daily { + request.Date = time.Date(2026, 5, 29, 0, 0, 0, 0, time.UTC) + } + resolved, err := report.DefaultRegistry().Resolve(id, request) if err != nil { t.Fatalf("Resolve() error = %v", err) }