Preserve generated text failure artifacts

2026-06-14 05:08:26 +00:00
parent 422430613c
commit 9e14a9b7c7
4 changed files with 356 additions and 8 deletions
--- a/internal/app/app.go
+++ b/internal/app/app.go
@@ -546,6 +546,9 @@ func GenerateReport(ctx context.Context, req ReportRequest) (*ReportResult, erro
 		return nil, metadataErr
 	}
 	if renderErr != nil {
+		if req.Resolved.Definition.GenerationMode == report.GenerationModeGeneratedTextTemplate {
+			return nil, generatedReportError(req.Resolved, metadata.RunID, "render preflight", renderErr)
+		}
 		return nil, renderErr
 	}

@@ -666,36 +669,44 @@ func generateTextTemplateReport(ctx context.Context, req generatedReportRequest)
 		}
 	}
 	if runErr != nil {
-		return nil, runErr
+		return nil, generatedReportError(req.Resolved, req.metadata.RunID, "structured generated text", runErr)
 	}

 	rawGeneratedText, err := req.store.LoadGeneratedText(ctx, req.paths.GeneratedTextRaw)
 	if err != nil {
-		return nil, err
+		return nil, generatedReportError(req.Resolved, req.metadata.RunID, "load raw generated text", err)
 	}
 	hourlyText, normalizedGeneratedText, err := validateGeneratedText(req.Resolved.Definition, rawGeneratedText)
 	if err != nil {
-		return nil, err
+		return nil, generatedReportError(req.Resolved, req.metadata.RunID, "validate generated text", err)
 	}
 	generatedTextPath, err := req.store.SaveGeneratedText(ctx, req.Resolved, normalizedGeneratedText)
 	if err != nil {
 		return nil, err
 	}
 	req.metadata.GeneratedTextPath = generatedTextPath
+	req.metadataPath, err = req.store.SaveMetadata(ctx, req.metadata)
+	if err != nil {
+		return nil, err
+	}

 	renderContext, err := buildRenderContext(req.Resolved.Definition, req.briefingMetadata, req.moduleSnapshot, hourlyText)
 	if err != nil {
-		return nil, err
+		return nil, generatedReportError(req.Resolved, req.metadata.RunID, "build render context", err)
 	}
 	renderContextPath, err := req.store.SaveRenderContext(ctx, req.Resolved, renderContext)
 	if err != nil {
 		return nil, err
 	}
 	req.metadata.RenderContextPath = renderContextPath
+	req.metadataPath, err = req.store.SaveMetadata(ctx, req.metadata)
+	if err != nil {
+		return nil, err
+	}

 	rendered, err := reporttemplate.Render(req.Resolved.Definition.TemplateID, renderContext)
 	if err != nil {
-		return nil, err
+		return nil, generatedReportError(req.Resolved, req.metadata.RunID, "render template", err)
 	}
 	reportPath, err := req.store.PrepareRenderedReport(ctx, req.Resolved)
 	if err != nil {
@@ -1084,10 +1095,17 @@ func validateGeneratedText(definition report.Definition, data []byte) (generated
 }

 func buildRenderContext(definition report.Definition, metadata briefing.Metadata, snapshot module.Snapshot, hourly generatedtext.Hourly) (any, error) {
-	switch definition.TemplateID {
+	switch definition.GeneratedTextSchemaID {
 	case "hourly":
 		return generatedtext.BuildHourlyRenderContext(metadata, snapshot, hourly)
 	default:
-		return nil, fmt.Errorf("report template %q is not supported for report %q", definition.TemplateID, definition.ID)
+		return nil, fmt.Errorf("generated text schema %q is not supported for report %q", definition.GeneratedTextSchemaID, definition.ID)
 	}
 }
+
+func generatedReportError(resolved report.Resolved, runID string, operation string, err error) error {
+	if err == nil {
+		return nil
+	}
+	return fmt.Errorf("generate report %q run %q: %s: %w", resolved.Definition.ID, runID, operation, err)
+}
--- a/internal/app/app_test.go
+++ b/internal/app/app_test.go
@@ -417,7 +417,7 @@ func TestGenerateHourlyReportUsesGeneratedTextTemplateWorkflow(t *testing.T) {
 	if renderer.structuredRunRequest.DataPackagePath != result.DataPackagePath {
 		t.Fatalf("structured run DataPackagePath = %q, want %q", renderer.structuredRunRequest.DataPackagePath, result.DataPackagePath)
 	}
-	if got, want := strings.Join(store.calls, ","), "module_snapshot,data_package,preflight,metadata,generated_text_result,metadata,generated_text,render_context,prepare_report,metadata"; got != want {
+	if got, want := strings.Join(store.calls, ","), "module_snapshot,data_package,preflight,metadata,generated_text_result,metadata,generated_text,metadata,render_context,metadata,prepare_report,metadata"; got != want {
 		t.Fatalf("store calls = %v, want %s", store.calls, want)
 	}

@@ -501,6 +501,212 @@ func TestGenerateHourlyReportUsesGeneratedTextTemplateWorkflow(t *testing.T) {
 	}
 }

+func TestGenerateHourlyReportPersistsPreflightFailure(t *testing.T) {
+	cfg, resolved, store, notifier, outputPath := hourlyGeneratedTextFixture(t)
+	renderer := &recordingRenderer{
+		renderResult: &scriptorium.RenderResult{
+			Command:  []string{"scriptorium", "render"},
+			Stderr:   "render failed",
+			ExitCode: 1,
+		},
+		err: errors.New("scriptorium render exited with code 1: render failed"),
+	}
+
+	_, err := GenerateReport(context.Background(), ReportRequest{
+		Config:     cfg,
+		Resolved:   resolved,
+		OutputPath: outputPath,
+		Renderer:   renderer,
+		Store:      store,
+		Notifier:   notifier,
+	})
+	assertGeneratedReportError(t, err, resolved, "render preflight")
+	if renderer.structuredRunCalls != 0 || renderer.runCalls != 0 {
+		t.Fatalf("post-preflight calls structured=%d run=%d, want none", renderer.structuredRunCalls, renderer.runCalls)
+	}
+	assertNoGeneratedFailureSideEffects(t, notifier, outputPath)
+
+	paths := hourlyArtifactPaths(t, store, resolved)
+	assertPathsExist(t, paths.Preflight, paths.Metadata)
+	assertPathsMissing(t, paths.GeneratedTextRaw, paths.GeneratedTextResult, paths.GeneratedText, paths.RenderContext, paths.RenderedReport)
+	preflight, readErr := os.ReadFile(paths.Preflight)
+	if readErr != nil {
+		t.Fatalf("read failed preflight: %v", readErr)
+	}
+	if !strings.Contains(string(preflight), `"exitCode": 1`) || !strings.Contains(string(preflight), "render failed") {
+		t.Fatalf("failed preflight was not persisted:\n%s", string(preflight))
+	}
+	metadataData, readErr := os.ReadFile(paths.Metadata)
+	if readErr != nil {
+		t.Fatalf("read metadata: %v", readErr)
+	}
+	if !strings.Contains(string(metadataData), paths.Preflight) || !strings.Contains(string(metadataData), paths.GeneratedTextRaw) {
+		t.Fatalf("metadata missing failed-run artifact links:\n%s", string(metadataData))
+	}
+}
+
+func TestGenerateHourlyReportPersistsStructuredRunFailure(t *testing.T) {
+	cfg, resolved, store, notifier, outputPath := hourlyGeneratedTextFixture(t)
+	renderer := &recordingRenderer{
+		renderResult: &scriptorium.RenderResult{ExitCode: 0},
+		structuredRunResult: &scriptorium.StructuredRunResult{
+			Command:  []string{"scriptorium", "run", "--json"},
+			Stderr:   "generation failed",
+			ExitCode: 2,
+		},
+		structuredRunErr:  errors.New("scriptorium structured run exited with code 2: generation failed"),
+		structuredRunBody: validHourlyGeneratedTextJSON(),
+	}
+
+	_, err := GenerateReport(context.Background(), ReportRequest{
+		Config:     cfg,
+		Resolved:   resolved,
+		OutputPath: outputPath,
+		Renderer:   renderer,
+		Store:      store,
+		Notifier:   notifier,
+	})
+	assertGeneratedReportError(t, err, resolved, "structured generated text")
+	if renderer.structuredRunCalls != 1 || renderer.runCalls != 0 {
+		t.Fatalf("calls structured=%d run=%d, want one structured run and no markdown run", renderer.structuredRunCalls, renderer.runCalls)
+	}
+	assertNoGeneratedFailureSideEffects(t, notifier, outputPath)
+
+	paths := hourlyArtifactPaths(t, store, resolved)
+	assertPathsExist(t, paths.Preflight, paths.Metadata, paths.GeneratedTextRaw, paths.GeneratedTextResult)
+	assertPathsMissing(t, paths.GeneratedText, paths.RenderContext, paths.RenderedReport)
+	metadataData, readErr := os.ReadFile(paths.Metadata)
+	if readErr != nil {
+		t.Fatalf("read metadata: %v", readErr)
+	}
+	if !strings.Contains(string(metadataData), paths.GeneratedTextRaw) || !strings.Contains(string(metadataData), paths.GeneratedTextResult) {
+		t.Fatalf("metadata missing structured failure links:\n%s", string(metadataData))
+	}
+}
+
+func TestGenerateHourlyReportPreservesRawTextOnValidationFailure(t *testing.T) {
+	cfg, resolved, store, notifier, outputPath := hourlyGeneratedTextFixture(t)
+	renderer := &recordingRenderer{
+		renderResult:        &scriptorium.RenderResult{ExitCode: 0},
+		structuredRunResult: &scriptorium.StructuredRunResult{ExitCode: 0},
+		structuredRunBody: `{
+			"summary": "Storm chances increase through late morning.",
+			"timing": "The main window is 10 AM to noon.",
+			"impacts": "Brief downpours may slow travel.",
+			"details": "not allowed"
+		}`,
+	}
+
+	_, err := GenerateReport(context.Background(), ReportRequest{
+		Config:     cfg,
+		Resolved:   resolved,
+		OutputPath: outputPath,
+		Renderer:   renderer,
+		Store:      store,
+		Notifier:   notifier,
+	})
+	assertGeneratedReportError(t, err, resolved, "validate generated text")
+	assertNoGeneratedFailureSideEffects(t, notifier, outputPath)
+
+	paths := hourlyArtifactPaths(t, store, resolved)
+	assertPathsExist(t, paths.Preflight, paths.Metadata, paths.GeneratedTextRaw, paths.GeneratedTextResult)
+	assertPathsMissing(t, paths.GeneratedText, paths.RenderContext, paths.RenderedReport)
+	raw, readErr := os.ReadFile(paths.GeneratedTextRaw)
+	if readErr != nil {
+		t.Fatalf("read raw generated text: %v", readErr)
+	}
+	if !strings.Contains(string(raw), `"details": "not allowed"`) {
+		t.Fatalf("raw generated text was not preserved:\n%s", string(raw))
+	}
+}
+
+func TestGenerateHourlyReportPreservesValidatedTextOnRenderContextFailure(t *testing.T) {
+	server := hourlyBundleServer(t)
+	cfg := hourlyGeneratedTextConfigWithModules(t, server, []string{
+		"metadata",
+		"hourly_forecast",
+		"precip_timing",
+		"alert_digest",
+		"area_forecast_discussion",
+		"weather_story",
+		"spc_convective_outlooks",
+	})
+	resolved, store, notifier, outputPath := resolveHourlyGeneratedTextFixture(t, cfg)
+	renderer := &recordingRenderer{
+		renderResult:        &scriptorium.RenderResult{ExitCode: 0},
+		structuredRunResult: &scriptorium.StructuredRunResult{ExitCode: 0},
+		structuredRunBody:   validHourlyGeneratedTextJSON(),
+	}
+
+	_, err := GenerateReport(context.Background(), ReportRequest{
+		Config:     cfg,
+		Resolved:   resolved,
+		OutputPath: outputPath,
+		Renderer:   renderer,
+		Store:      store,
+		Notifier:   notifier,
+	})
+	assertGeneratedReportError(t, err, resolved, "build render context")
+	assertNoGeneratedFailureSideEffects(t, notifier, outputPath)
+
+	paths := hourlyArtifactPaths(t, store, resolved)
+	assertPathsExist(t, paths.Preflight, paths.Metadata, paths.GeneratedTextRaw, paths.GeneratedTextResult, paths.GeneratedText)
+	assertPathsMissing(t, paths.RenderContext, paths.RenderedReport)
+	generatedText, readErr := os.ReadFile(paths.GeneratedText)
+	if readErr != nil {
+		t.Fatalf("read validated generated text: %v", readErr)
+	}
+	if string(generatedText) != validHourlyGeneratedTextJSON() {
+		t.Fatalf("validated generated text = %s, want normalized JSON", generatedText)
+	}
+	metadataData, readErr := os.ReadFile(paths.Metadata)
+	if readErr != nil {
+		t.Fatalf("read metadata: %v", readErr)
+	}
+	if !strings.Contains(string(metadataData), paths.GeneratedText) {
+		t.Fatalf("metadata missing validated generated text link:\n%s", string(metadataData))
+	}
+}
+
+func TestGenerateHourlyReportPreservesRenderContextOnTemplateFailure(t *testing.T) {
+	cfg, resolved, store, notifier, outputPath := hourlyGeneratedTextFixture(t)
+	resolved.Definition.TemplateID = "missing-template"
+	renderer := &recordingRenderer{
+		renderResult:        &scriptorium.RenderResult{ExitCode: 0},
+		structuredRunResult: &scriptorium.StructuredRunResult{ExitCode: 0},
+		structuredRunBody:   validHourlyGeneratedTextJSON(),
+	}
+
+	_, err := GenerateReport(context.Background(), ReportRequest{
+		Config:     cfg,
+		Resolved:   resolved,
+		OutputPath: outputPath,
+		Renderer:   renderer,
+		Store:      store,
+		Notifier:   notifier,
+	})
+	assertGeneratedReportError(t, err, resolved, "render template")
+	assertNoGeneratedFailureSideEffects(t, notifier, outputPath)
+
+	paths := hourlyArtifactPaths(t, store, resolved)
+	assertPathsExist(t, paths.Preflight, paths.Metadata, paths.GeneratedTextRaw, paths.GeneratedTextResult, paths.GeneratedText, paths.RenderContext)
+	assertPathsMissing(t, paths.RenderedReport)
+	renderContext, readErr := os.ReadFile(paths.RenderContext)
+	if readErr != nil {
+		t.Fatalf("read render context: %v", readErr)
+	}
+	if !strings.Contains(string(renderContext), `"ReportTitle": "Hourly Report"`) {
+		t.Fatalf("render context was not preserved:\n%s", string(renderContext))
+	}
+	metadataData, readErr := os.ReadFile(paths.Metadata)
+	if readErr != nil {
+		t.Fatalf("read metadata: %v", readErr)
+	}
+	if !strings.Contains(string(metadataData), paths.RenderContext) {
+		t.Fatalf("metadata missing render context link:\n%s", string(metadataData))
+	}
+}
+
 func TestGenerateReportDisabledNotificationDoesNotCallNotifier(t *testing.T) {
 	server := dailyBundleServer(t)
 	cfg := dailyTestConfig(t, server)
@@ -1686,6 +1892,75 @@ func dailyTestConfig(t *testing.T, server *httptest.Server) config.Config {
 	return cfg
 }

+func hourlyGeneratedTextFixture(t *testing.T) (config.Config, report.Resolved, *recordingStore, *recordingNotifier, string) {
+	t.Helper()
+	server := hourlyBundleServer(t)
+	cfg := hourlyGeneratedTextConfig(t, server)
+	resolved, store, notifier, outputPath := resolveHourlyGeneratedTextFixture(t, cfg)
+	return cfg, resolved, store, notifier, outputPath
+}
+
+func hourlyGeneratedTextConfig(t *testing.T, server *httptest.Server) config.Config {
+	t.Helper()
+	cfg := config.Defaults()
+	applyHourlyGeneratedTextSettings(&cfg, t, server)
+	return cfg
+}
+
+func hourlyGeneratedTextConfigWithModules(t *testing.T, server *httptest.Server, modules []string) config.Config {
+	t.Helper()
+	var data strings.Builder
+	data.WriteString("reports:\n  hourly:\n    deterministic_modules:\n")
+	for _, id := range modules {
+		_, _ = fmt.Fprintf(&data, "      - %s\n", id)
+	}
+	path := filepath.Join(t.TempDir(), "config.yml")
+	if err := os.WriteFile(path, []byte(data.String()), 0o600); err != nil {
+		t.Fatalf("write config fixture: %v", err)
+	}
+	cfg, err := config.LoadFile(path)
+	if err != nil {
+		t.Fatalf("LoadFile() error = %v", err)
+	}
+	applyHourlyGeneratedTextSettings(&cfg, t, server)
+	return cfg
+}
+
+func applyHourlyGeneratedTextSettings(cfg *config.Config, t *testing.T, server *httptest.Server) {
+	t.Helper()
+	cfg.WeatherAPI.BaseURL = server.URL + "/"
+	cfg.WeatherAPI.Timezone = "America/Chicago"
+	cfg.Workspace.Root = t.TempDir()
+	cfg.Location.ID = "home"
+	cfg.Location.Name = "Brentwood"
+	cfg.Location.Region = "MO"
+	cfg.Notify.Distributor.Enabled = true
+	cfg.Notify.Distributor.PipelineIDTemplate = "weatherreporter.{artifact_group}"
+	cfg.Notify.Distributor.BundleIDTemplate = "weatherreporter.{location_id}.{report_id}"
+	cfg.Notify.Distributor.IdempotencyKeyTemplate = "weatherreporter.{location_id}.{report_id}.{run_id}"
+	cfg.Notify.Distributor.ReportPathTemplates = []string{"{valid_start_date}/{artifact_group}/{batch_output_name}"}
+}
+
+func resolveHourlyGeneratedTextFixture(t *testing.T, cfg config.Config) (report.Resolved, *recordingStore, *recordingNotifier, string) {
+	t.Helper()
+	resolved, err := ResolveGenerate(GenerateRequest{
+		Config: cfg,
+		Report: ReportHourly,
+	}, mustParse("2026-05-29T08:30:00-05:00"))
+	if err != nil {
+		t.Fatalf("ResolveGenerate() error = %v", err)
+	}
+	filesystemStore, err := state.NewFilesystemStore(cfg.Workspace)
+	if err != nil {
+		t.Fatalf("NewFilesystemStore() error = %v", err)
+	}
+	return resolved, &recordingStore{Store: filesystemStore}, &recordingNotifier{}, filepath.Join(t.TempDir(), "hourly-copy.md")
+}
+
+func validHourlyGeneratedTextJSON() string {
+	return `{"summary":"Storm chances increase through late morning.","timing":"The main window is 10 AM to noon.","impacts":"Brief downpours may slow travel.","confidence":"Medium"}`
+}
+
 func generateDailyReportForTest(t *testing.T, cfg config.Config) *ReportResult {
 	t.Helper()
 	cfg.Workspace.Root = t.TempDir()
@@ -1754,6 +2029,51 @@ func assertPathsExist(t *testing.T, paths ...string) {
 	}
 }

+func assertPathsMissing(t *testing.T, paths ...string) {
+	t.Helper()
+	for _, path := range paths {
+		if _, err := os.Stat(path); err == nil {
+			t.Fatalf("artifact %q exists, want missing", path)
+		} else if !os.IsNotExist(err) {
+			t.Fatalf("stat artifact %q: %v", path, err)
+		}
+	}
+}
+
+func hourlyArtifactPaths(t *testing.T, store state.Store, resolved report.Resolved) state.ArtifactPaths {
+	t.Helper()
+	paths, err := store.Paths(resolved)
+	if err != nil {
+		t.Fatalf("Paths() error = %v", err)
+	}
+	return paths
+}
+
+func assertGeneratedReportError(t *testing.T, err error, resolved report.Resolved, operation string) {
+	t.Helper()
+	if err == nil {
+		t.Fatal("GenerateReport() error = nil, want generated-text report error")
+	}
+	text := err.Error()
+	for _, want := range []string{
+		fmt.Sprintf("generate report %q", resolved.Definition.ID),
+		fmt.Sprintf("run %q", resolved.Metadata().RunID),
+		operation,
+	} {
+		if !strings.Contains(text, want) {
+			t.Fatalf("error = %q, want %q", text, want)
+		}
+	}
+}
+
+func assertNoGeneratedFailureSideEffects(t *testing.T, notifier *recordingNotifier, outputPath string) {
+	t.Helper()
+	if len(notifier.requests) != 0 {
+		t.Fatalf("notification requests = %#v, want none after generated-text failure", notifier.requests)
+	}
+	assertPathsMissing(t, outputPath)
+}
+
 func savePriorRun(t *testing.T, store state.Store, resolved report.Resolved, snapshot module.Snapshot) {
 	t.Helper()
 	moduleSnapshotPath, err := store.SaveModuleSnapshot(context.Background(), resolved, snapshot)