Preserve generated text failure artifacts

docs/internal/app-orchestration.md

@@ -104,6 +104,10 @@ generation returns an error after writing output, the managed report and
 metadata remain inspectable. Notification is not attempted after Weather API,
 module snapshot, prompt input, render, Scriptorium run, or metadata-save
 failures.
+Generated-text report failures are returned with report ID, RunID, and the
+failed operation. When available, the app preserves the latest generated-text
+artifacts already reached by the workflow: preflight output, structured run
+result, raw generated text, validated generated text, and render context.
 When notification is attempted, the debug artifact records request identity,
 including rendered pipeline ID, bundle paths, accepted upload fields,
 distributor status fields, raw status report JSON when available, and redacted
@@ -136,6 +140,8 @@ inspection view.
   runs.
 - Prompt input validation fails before render preflight.
 - Render and run errors preserve Scriptorium stderr and exit-code context.
+- Generated-text report errors preserve available intermediate artifacts and do
+  not create extra output copies.
 - Notification errors are wrapped with report ID, RunID, and managed report path
   context and are recorded separately in batch results.
 - Metadata and artifact path errors include filesystem context.

docs/operations.md

@@ -262,6 +262,10 @@ A failed generation run may still leave useful artifacts:
   preflight JSON and metadata are written for inspection.
 - If `scriptorium run` exits nonzero after writing a report, the managed report
   and metadata remain available.
+- Hourly generated-text failures preserve available intermediate artifacts,
+  such as the structured run result, raw generated-text JSON, validated
+  generated text, and render context. Metadata links those paths when it can be
+  safely written.
 - If distributor notification fails, report artifacts and final metadata remain
   available, but the report or batch command returns nonzero.
 - For batch commands, inspect the stdout JSON summary first, then inspect the

internal/app/app.go

@@ -546,6 +546,9 @@ func GenerateReport(ctx context.Context, req ReportRequest) (*ReportResult, erro
 		return nil, metadataErr
 	}
 	if renderErr != nil {
+		if req.Resolved.Definition.GenerationMode == report.GenerationModeGeneratedTextTemplate {
+			return nil, generatedReportError(req.Resolved, metadata.RunID, "render preflight", renderErr)
+		}
 		return nil, renderErr
 	}
 
@@ -666,36 +669,44 @@ func generateTextTemplateReport(ctx context.Context, req generatedReportRequest)
 		}
 	}
 	if runErr != nil {
-		return nil, runErr
+		return nil, generatedReportError(req.Resolved, req.metadata.RunID, "structured generated text", runErr)
 	}
 
 	rawGeneratedText, err := req.store.LoadGeneratedText(ctx, req.paths.GeneratedTextRaw)
 	if err != nil {
-		return nil, err
+		return nil, generatedReportError(req.Resolved, req.metadata.RunID, "load raw generated text", err)
 	}
 	hourlyText, normalizedGeneratedText, err := validateGeneratedText(req.Resolved.Definition, rawGeneratedText)
 	if err != nil {
-		return nil, err
+		return nil, generatedReportError(req.Resolved, req.metadata.RunID, "validate generated text", err)
 	}
 	generatedTextPath, err := req.store.SaveGeneratedText(ctx, req.Resolved, normalizedGeneratedText)
 	if err != nil {
 		return nil, err
 	}
 	req.metadata.GeneratedTextPath = generatedTextPath
+	req.metadataPath, err = req.store.SaveMetadata(ctx, req.metadata)
+	if err != nil {
+		return nil, err
+	}
 
 	renderContext, err := buildRenderContext(req.Resolved.Definition, req.briefingMetadata, req.moduleSnapshot, hourlyText)
 	if err != nil {
-		return nil, err
+		return nil, generatedReportError(req.Resolved, req.metadata.RunID, "build render context", err)
 	}
 	renderContextPath, err := req.store.SaveRenderContext(ctx, req.Resolved, renderContext)
 	if err != nil {
 		return nil, err
 	}
 	req.metadata.RenderContextPath = renderContextPath
+	req.metadataPath, err = req.store.SaveMetadata(ctx, req.metadata)
+	if err != nil {
+		return nil, err
+	}
 
 	rendered, err := reporttemplate.Render(req.Resolved.Definition.TemplateID, renderContext)
 	if err != nil {
-		return nil, err
+		return nil, generatedReportError(req.Resolved, req.metadata.RunID, "render template", err)
 	}
 	reportPath, err := req.store.PrepareRenderedReport(ctx, req.Resolved)
 	if err != nil {
@@ -1084,10 +1095,17 @@ func validateGeneratedText(definition report.Definition, data []byte) (generated
 }
 
 func buildRenderContext(definition report.Definition, metadata briefing.Metadata, snapshot module.Snapshot, hourly generatedtext.Hourly) (any, error) {
-	switch definition.TemplateID {
+	switch definition.GeneratedTextSchemaID {
 	case "hourly":
 		return generatedtext.BuildHourlyRenderContext(metadata, snapshot, hourly)
 	default:
-		return nil, fmt.Errorf("report template %q is not supported for report %q", definition.TemplateID, definition.ID)
+		return nil, fmt.Errorf("generated text schema %q is not supported for report %q", definition.GeneratedTextSchemaID, definition.ID)
 	}
 }
+
+func generatedReportError(resolved report.Resolved, runID string, operation string, err error) error {
+	if err == nil {
+		return nil
+	}
+	return fmt.Errorf("generate report %q run %q: %s: %w", resolved.Definition.ID, runID, operation, err)
+}

internal/app/app_test.go

@@ -417,7 +417,7 @@ func TestGenerateHourlyReportUsesGeneratedTextTemplateWorkflow(t *testing.T) {
 	if renderer.structuredRunRequest.DataPackagePath != result.DataPackagePath {
 		t.Fatalf("structured run DataPackagePath = %q, want %q", renderer.structuredRunRequest.DataPackagePath, result.DataPackagePath)
 	}
-	if got, want := strings.Join(store.calls, ","), "module_snapshot,data_package,preflight,metadata,generated_text_result,metadata,generated_text,render_context,prepare_report,metadata"; got != want {
+	if got, want := strings.Join(store.calls, ","), "module_snapshot,data_package,preflight,metadata,generated_text_result,metadata,generated_text,metadata,render_context,metadata,prepare_report,metadata"; got != want {
 		t.Fatalf("store calls = %v, want %s", store.calls, want)
 	}
 
@@ -501,6 +501,212 @@ func TestGenerateHourlyReportUsesGeneratedTextTemplateWorkflow(t *testing.T) {
 	}
 }
 
+func TestGenerateHourlyReportPersistsPreflightFailure(t *testing.T) {
+	cfg, resolved, store, notifier, outputPath := hourlyGeneratedTextFixture(t)
+	renderer := &recordingRenderer{
+		renderResult: &scriptorium.RenderResult{
+			Command:  []string{"scriptorium", "render"},
+			Stderr:   "render failed",
+			ExitCode: 1,
+		},
+		err: errors.New("scriptorium render exited with code 1: render failed"),
+	}
+
+	_, err := GenerateReport(context.Background(), ReportRequest{
+		Config:     cfg,
+		Resolved:   resolved,
+		OutputPath: outputPath,
+		Renderer:   renderer,
+		Store:      store,
+		Notifier:   notifier,
+	})
+	assertGeneratedReportError(t, err, resolved, "render preflight")
+	if renderer.structuredRunCalls != 0 || renderer.runCalls != 0 {
+		t.Fatalf("post-preflight calls structured=%d run=%d, want none", renderer.structuredRunCalls, renderer.runCalls)
+	}
+	assertNoGeneratedFailureSideEffects(t, notifier, outputPath)
+
+	paths := hourlyArtifactPaths(t, store, resolved)
+	assertPathsExist(t, paths.Preflight, paths.Metadata)
+	assertPathsMissing(t, paths.GeneratedTextRaw, paths.GeneratedTextResult, paths.GeneratedText, paths.RenderContext, paths.RenderedReport)
+	preflight, readErr := os.ReadFile(paths.Preflight)
+	if readErr != nil {
+		t.Fatalf("read failed preflight: %v", readErr)
+	}
+	if !strings.Contains(string(preflight), `"exitCode": 1`) || !strings.Contains(string(preflight), "render failed") {
+		t.Fatalf("failed preflight was not persisted:\n%s", string(preflight))
+	}
+	metadataData, readErr := os.ReadFile(paths.Metadata)
+	if readErr != nil {
+		t.Fatalf("read metadata: %v", readErr)
+	}
+	if !strings.Contains(string(metadataData), paths.Preflight) || !strings.Contains(string(metadataData), paths.GeneratedTextRaw) {
+		t.Fatalf("metadata missing failed-run artifact links:\n%s", string(metadataData))
+	}
+}
+
+func TestGenerateHourlyReportPersistsStructuredRunFailure(t *testing.T) {
+	cfg, resolved, store, notifier, outputPath := hourlyGeneratedTextFixture(t)
+	renderer := &recordingRenderer{
+		renderResult: &scriptorium.RenderResult{ExitCode: 0},
+		structuredRunResult: &scriptorium.StructuredRunResult{
+			Command:  []string{"scriptorium", "run", "--json"},
+			Stderr:   "generation failed",
+			ExitCode: 2,
+		},
+		structuredRunErr:  errors.New("scriptorium structured run exited with code 2: generation failed"),
+		structuredRunBody: validHourlyGeneratedTextJSON(),
+	}
+
+	_, err := GenerateReport(context.Background(), ReportRequest{
+		Config:     cfg,
+		Resolved:   resolved,
+		OutputPath: outputPath,
+		Renderer:   renderer,
+		Store:      store,
+		Notifier:   notifier,
+	})
+	assertGeneratedReportError(t, err, resolved, "structured generated text")
+	if renderer.structuredRunCalls != 1 || renderer.runCalls != 0 {
+		t.Fatalf("calls structured=%d run=%d, want one structured run and no markdown run", renderer.structuredRunCalls, renderer.runCalls)
+	}
+	assertNoGeneratedFailureSideEffects(t, notifier, outputPath)
+
+	paths := hourlyArtifactPaths(t, store, resolved)
+	assertPathsExist(t, paths.Preflight, paths.Metadata, paths.GeneratedTextRaw, paths.GeneratedTextResult)
+	assertPathsMissing(t, paths.GeneratedText, paths.RenderContext, paths.RenderedReport)
+	metadataData, readErr := os.ReadFile(paths.Metadata)
+	if readErr != nil {
+		t.Fatalf("read metadata: %v", readErr)
+	}
+	if !strings.Contains(string(metadataData), paths.GeneratedTextRaw) || !strings.Contains(string(metadataData), paths.GeneratedTextResult) {
+		t.Fatalf("metadata missing structured failure links:\n%s", string(metadataData))
+	}
+}
+
+func TestGenerateHourlyReportPreservesRawTextOnValidationFailure(t *testing.T) {
+	cfg, resolved, store, notifier, outputPath := hourlyGeneratedTextFixture(t)
+	renderer := &recordingRenderer{
+		renderResult:        &scriptorium.RenderResult{ExitCode: 0},
+		structuredRunResult: &scriptorium.StructuredRunResult{ExitCode: 0},
+		structuredRunBody: `{
+			"summary": "Storm chances increase through late morning.",
+			"timing": "The main window is 10 AM to noon.",
+			"impacts": "Brief downpours may slow travel.",
+			"details": "not allowed"
+		}`,
+	}
+
+	_, err := GenerateReport(context.Background(), ReportRequest{
+		Config:     cfg,
+		Resolved:   resolved,
+		OutputPath: outputPath,
+		Renderer:   renderer,
+		Store:      store,
+		Notifier:   notifier,
+	})
+	assertGeneratedReportError(t, err, resolved, "validate generated text")
+	assertNoGeneratedFailureSideEffects(t, notifier, outputPath)
+
+	paths := hourlyArtifactPaths(t, store, resolved)
+	assertPathsExist(t, paths.Preflight, paths.Metadata, paths.GeneratedTextRaw, paths.GeneratedTextResult)
+	assertPathsMissing(t, paths.GeneratedText, paths.RenderContext, paths.RenderedReport)
+	raw, readErr := os.ReadFile(paths.GeneratedTextRaw)
+	if readErr != nil {
+		t.Fatalf("read raw generated text: %v", readErr)
+	}
+	if !strings.Contains(string(raw), `"details": "not allowed"`) {
+		t.Fatalf("raw generated text was not preserved:\n%s", string(raw))
+	}
+}
+
+func TestGenerateHourlyReportPreservesValidatedTextOnRenderContextFailure(t *testing.T) {
+	server := hourlyBundleServer(t)
+	cfg := hourlyGeneratedTextConfigWithModules(t, server, []string{
+		"metadata",
+		"hourly_forecast",
+		"precip_timing",
+		"alert_digest",
+		"area_forecast_discussion",
+		"weather_story",
+		"spc_convective_outlooks",
+	})
+	resolved, store, notifier, outputPath := resolveHourlyGeneratedTextFixture(t, cfg)
+	renderer := &recordingRenderer{
+		renderResult:        &scriptorium.RenderResult{ExitCode: 0},
+		structuredRunResult: &scriptorium.StructuredRunResult{ExitCode: 0},
+		structuredRunBody:   validHourlyGeneratedTextJSON(),
+	}
+
+	_, err := GenerateReport(context.Background(), ReportRequest{
+		Config:     cfg,
+		Resolved:   resolved,
+		OutputPath: outputPath,
+		Renderer:   renderer,
+		Store:      store,
+		Notifier:   notifier,
+	})
+	assertGeneratedReportError(t, err, resolved, "build render context")
+	assertNoGeneratedFailureSideEffects(t, notifier, outputPath)
+
+	paths := hourlyArtifactPaths(t, store, resolved)
+	assertPathsExist(t, paths.Preflight, paths.Metadata, paths.GeneratedTextRaw, paths.GeneratedTextResult, paths.GeneratedText)
+	assertPathsMissing(t, paths.RenderContext, paths.RenderedReport)
+	generatedText, readErr := os.ReadFile(paths.GeneratedText)
+	if readErr != nil {
+		t.Fatalf("read validated generated text: %v", readErr)
+	}
+	if string(generatedText) != validHourlyGeneratedTextJSON() {
+		t.Fatalf("validated generated text = %s, want normalized JSON", generatedText)
+	}
+	metadataData, readErr := os.ReadFile(paths.Metadata)
+	if readErr != nil {
+		t.Fatalf("read metadata: %v", readErr)
+	}
+	if !strings.Contains(string(metadataData), paths.GeneratedText) {
+		t.Fatalf("metadata missing validated generated text link:\n%s", string(metadataData))
+	}
+}
+
+func TestGenerateHourlyReportPreservesRenderContextOnTemplateFailure(t *testing.T) {
+	cfg, resolved, store, notifier, outputPath := hourlyGeneratedTextFixture(t)
+	resolved.Definition.TemplateID = "missing-template"
+	renderer := &recordingRenderer{
+		renderResult:        &scriptorium.RenderResult{ExitCode: 0},
+		structuredRunResult: &scriptorium.StructuredRunResult{ExitCode: 0},
+		structuredRunBody:   validHourlyGeneratedTextJSON(),
+	}
+
+	_, err := GenerateReport(context.Background(), ReportRequest{
+		Config:     cfg,
+		Resolved:   resolved,
+		OutputPath: outputPath,
+		Renderer:   renderer,
+		Store:      store,
+		Notifier:   notifier,
+	})
+	assertGeneratedReportError(t, err, resolved, "render template")
+	assertNoGeneratedFailureSideEffects(t, notifier, outputPath)
+
+	paths := hourlyArtifactPaths(t, store, resolved)
+	assertPathsExist(t, paths.Preflight, paths.Metadata, paths.GeneratedTextRaw, paths.GeneratedTextResult, paths.GeneratedText, paths.RenderContext)
+	assertPathsMissing(t, paths.RenderedReport)
+	renderContext, readErr := os.ReadFile(paths.RenderContext)
+	if readErr != nil {
+		t.Fatalf("read render context: %v", readErr)
+	}
+	if !strings.Contains(string(renderContext), `"ReportTitle": "Hourly Report"`) {
+		t.Fatalf("render context was not preserved:\n%s", string(renderContext))
+	}
+	metadataData, readErr := os.ReadFile(paths.Metadata)
+	if readErr != nil {
+		t.Fatalf("read metadata: %v", readErr)
+	}
+	if !strings.Contains(string(metadataData), paths.RenderContext) {
+		t.Fatalf("metadata missing render context link:\n%s", string(metadataData))
+	}
+}
+
 func TestGenerateReportDisabledNotificationDoesNotCallNotifier(t *testing.T) {
 	server := dailyBundleServer(t)
 	cfg := dailyTestConfig(t, server)
@@ -1686,6 +1892,75 @@ func dailyTestConfig(t *testing.T, server *httptest.Server) config.Config {
 	return cfg
 }
 
+func hourlyGeneratedTextFixture(t *testing.T) (config.Config, report.Resolved, *recordingStore, *recordingNotifier, string) {
+	t.Helper()
+	server := hourlyBundleServer(t)
+	cfg := hourlyGeneratedTextConfig(t, server)
+	resolved, store, notifier, outputPath := resolveHourlyGeneratedTextFixture(t, cfg)
+	return cfg, resolved, store, notifier, outputPath
+}
+
+func hourlyGeneratedTextConfig(t *testing.T, server *httptest.Server) config.Config {
+	t.Helper()
+	cfg := config.Defaults()
+	applyHourlyGeneratedTextSettings(&cfg, t, server)
+	return cfg
+}
+
+func hourlyGeneratedTextConfigWithModules(t *testing.T, server *httptest.Server, modules []string) config.Config {
+	t.Helper()
+	var data strings.Builder
+	data.WriteString("reports:\n  hourly:\n    deterministic_modules:\n")
+	for _, id := range modules {
+		_, _ = fmt.Fprintf(&data, "      - %s\n", id)
+	}
+	path := filepath.Join(t.TempDir(), "config.yml")
+	if err := os.WriteFile(path, []byte(data.String()), 0o600); err != nil {
+		t.Fatalf("write config fixture: %v", err)
+	}
+	cfg, err := config.LoadFile(path)
+	if err != nil {
+		t.Fatalf("LoadFile() error = %v", err)
+	}
+	applyHourlyGeneratedTextSettings(&cfg, t, server)
+	return cfg
+}
+
+func applyHourlyGeneratedTextSettings(cfg *config.Config, t *testing.T, server *httptest.Server) {
+	t.Helper()
+	cfg.WeatherAPI.BaseURL = server.URL + "/"
+	cfg.WeatherAPI.Timezone = "America/Chicago"
+	cfg.Workspace.Root = t.TempDir()
+	cfg.Location.ID = "home"
+	cfg.Location.Name = "Brentwood"
+	cfg.Location.Region = "MO"
+	cfg.Notify.Distributor.Enabled = true
+	cfg.Notify.Distributor.PipelineIDTemplate = "weatherreporter.{artifact_group}"
+	cfg.Notify.Distributor.BundleIDTemplate = "weatherreporter.{location_id}.{report_id}"
+	cfg.Notify.Distributor.IdempotencyKeyTemplate = "weatherreporter.{location_id}.{report_id}.{run_id}"
+	cfg.Notify.Distributor.ReportPathTemplates = []string{"{valid_start_date}/{artifact_group}/{batch_output_name}"}
+}
+
+func resolveHourlyGeneratedTextFixture(t *testing.T, cfg config.Config) (report.Resolved, *recordingStore, *recordingNotifier, string) {
+	t.Helper()
+	resolved, err := ResolveGenerate(GenerateRequest{
+		Config: cfg,
+		Report: ReportHourly,
+	}, mustParse("2026-05-29T08:30:00-05:00"))
+	if err != nil {
+		t.Fatalf("ResolveGenerate() error = %v", err)
+	}
+	filesystemStore, err := state.NewFilesystemStore(cfg.Workspace)
+	if err != nil {
+		t.Fatalf("NewFilesystemStore() error = %v", err)
+	}
+	return resolved, &recordingStore{Store: filesystemStore}, &recordingNotifier{}, filepath.Join(t.TempDir(), "hourly-copy.md")
+}
+
+func validHourlyGeneratedTextJSON() string {
+	return `{"summary":"Storm chances increase through late morning.","timing":"The main window is 10 AM to noon.","impacts":"Brief downpours may slow travel.","confidence":"Medium"}`
+}
+
 func generateDailyReportForTest(t *testing.T, cfg config.Config) *ReportResult {
 	t.Helper()
 	cfg.Workspace.Root = t.TempDir()
@@ -1754,6 +2029,51 @@ func assertPathsExist(t *testing.T, paths ...string) {
 	}
 }
 
+func assertPathsMissing(t *testing.T, paths ...string) {
+	t.Helper()
+	for _, path := range paths {
+		if _, err := os.Stat(path); err == nil {
+			t.Fatalf("artifact %q exists, want missing", path)
+		} else if !os.IsNotExist(err) {
+			t.Fatalf("stat artifact %q: %v", path, err)
+		}
+	}
+}
+
+func hourlyArtifactPaths(t *testing.T, store state.Store, resolved report.Resolved) state.ArtifactPaths {
+	t.Helper()
+	paths, err := store.Paths(resolved)
+	if err != nil {
+		t.Fatalf("Paths() error = %v", err)
+	}
+	return paths
+}
+
+func assertGeneratedReportError(t *testing.T, err error, resolved report.Resolved, operation string) {
+	t.Helper()
+	if err == nil {
+		t.Fatal("GenerateReport() error = nil, want generated-text report error")
+	}
+	text := err.Error()
+	for _, want := range []string{
+		fmt.Sprintf("generate report %q", resolved.Definition.ID),
+		fmt.Sprintf("run %q", resolved.Metadata().RunID),
+		operation,
+	} {
+		if !strings.Contains(text, want) {
+			t.Fatalf("error = %q, want %q", text, want)
+		}
+	}
+}
+
+func assertNoGeneratedFailureSideEffects(t *testing.T, notifier *recordingNotifier, outputPath string) {
+	t.Helper()
+	if len(notifier.requests) != 0 {
+		t.Fatalf("notification requests = %#v, want none after generated-text failure", notifier.requests)
+	}
+	assertPathsMissing(t, outputPath)
+}
+
 func savePriorRun(t *testing.T, store state.Store, resolved report.Resolved, snapshot module.Snapshot) {
 	t.Helper()
 	moduleSnapshotPath, err := store.SaveModuleSnapshot(context.Background(), resolved, snapshot)