diff --git a/docs/cli.md b/docs/cli.md index 700432f..d0de5c1 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -93,6 +93,7 @@ Valid stage names: - `polish` - `normalize` - `trim` +- `render` - `analyze` - `publish` - `notify` diff --git a/docs/config.md b/docs/config.md index 648dfaf..64e4480 100644 --- a/docs/config.md +++ b/docs/config.md @@ -98,6 +98,12 @@ publish: - source: narratio.transcript.final_trimmed dest: transcripts/final.trimmed.json required: true + - source: narratio.transcript.final_markdown + dest: transcripts/final.md + required: true + - source: narratio.transcript.final_trimmed_markdown + dest: transcripts/final.trimmed.md + required: true - source: narratio.artifact.session_recap dest: artifacts/session_recap.md required: true @@ -138,7 +144,7 @@ Rules: | `pipeline.cache.s3_audio` | bool | No | `true` | | `pipeline.publish.enabled` | bool | No | `true` | | `pipeline.publish.upload_run` | bool | No | `true` | -| `pipeline.publish.outputs[]` | list | No | defaults to final trimmed transcript output | +| `pipeline.publish.outputs[]` | list | No | defaults to final trimmed JSON plus final and final-trimmed Markdown outputs | | `pipeline.publish.outputs[].source` | string | Yes (per rule) | must reference built-in or configured artifact source | | `pipeline.publish.outputs[].dest` | string | Conditional | derived if omitted and source supports derivation | | `pipeline.publish.outputs[].required` | bool | No | `true` | @@ -188,6 +194,12 @@ Rules: | `pipeline.trim.bounds.render_debug` | bool | No | `false` | | `pipeline.trim.bounds.render_output_path` | string | Conditional | required when `render_debug` is true | | `pipeline.trim.seriatim.report` | bool | No | `false` | +| `pipeline.render.enabled` | bool | No | `true` | +| `pipeline.render.format` | string | No | `markdown` (only supported value) | +| `pipeline.render.title` | string | No | empty (falls back to `session.title` when set) | +| `pipeline.render.include_timestamps` | bool | No | `true` | +| `pipeline.render.include_segment_ids` | bool | No | `false` | +| `pipeline.render.include_metadata` | bool | No | `false` | | `pipeline.scriptorium.binary` | string | No | `scriptorium` | | `pipeline.scriptorium.config_path` | string | No | empty | | `pipeline.scriptorium.timeout` | duration | No | `10m` | diff --git a/docs/integrations/seriatim.md b/docs/integrations/seriatim.md index 491c341..33821a1 100644 --- a/docs/integrations/seriatim.md +++ b/docs/integrations/seriatim.md @@ -1,7 +1,7 @@ # Integration: Seriatim ## Purpose -Define the Seriatim adapter contract used by `merge`, `normalize`, and `trim`. +Define the Seriatim adapter contract used by `merge`, `normalize`, `trim`, and `render`. ## Adapter Boundary Interface: @@ -10,6 +10,7 @@ Interface: - `Run(ctx, MergeRequest)` - `Normalize(ctx, NormalizeRequest)` - `Trim(ctx, TrimRequest)` + - `Render(ctx, RenderRequest)` Primary implementation: - `internal/adapters/seriatim/SubprocessRunner` @@ -18,11 +19,13 @@ Execution modes: - `seriatim merge` - `seriatim normalize` - `seriatim trim` +- `seriatim render` ## Request/Result Contracts - `MergeRequest`/`MergeResult`: multi-input merge to base transcript, optional report. - `NormalizeRequest`/`NormalizeResult`: transcript normalization with explicit schema. - `TrimRequest`/`TrimResult`: transcript trimming with required keep selector. +- `RenderRequest`/`RenderResult`: transcript-to-markdown rendering with explicit format and render booleans. Results include output/log/config paths, timing, exit code, and metadata. @@ -36,9 +39,11 @@ Runner construction validates: Invocation fails on: - missing required request paths/inputs; - invalid normalize schema override; +- unsupported render format; - subprocess failure; -- invalid JSON outputs; -- missing `segments` array for normalize/trim transcript outputs. +- invalid JSON outputs for merge/normalize/trim; +- missing `segments` array for normalize/trim transcript outputs; +- empty render output files. When report paths are provided/enabled, report files must parse as JSON. @@ -49,7 +54,7 @@ When report paths are provided/enabled, report files must parse as JSON. - adapter does not write manifests or choose stage inputs. ## Config Mapping -Config fields consumed through runner/stage wiring are under `pipeline.seriatim.*`. +Config fields consumed through runner/stage wiring are under `pipeline.seriatim.*` and `pipeline.render.*`. Maintained examples with Seriatim config: - `examples/pipeline.full.annotated.yml` diff --git a/docs/internal/README.md b/docs/internal/README.md index 0c0f34f..5c12387 100644 --- a/docs/internal/README.md +++ b/docs/internal/README.md @@ -20,9 +20,10 @@ Canonical stage order from `internal/stage.All()`: 4. `polish` 5. `normalize` 6. `trim` -7. `analyze` -8. `publish` -9. `notify` (placeholder) +7. `render` +8. `analyze` +9. `publish` +10. `notify` (placeholder) `notify` is currently a placeholder stage with optional notifier call behavior; it has no persisted pipeline outputs. @@ -39,5 +40,6 @@ Canonical stage order from `internal/stage.All()`: - `stage-polish.md` - `stage-normalize.md` - `stage-trim.md` +- `stage-render.md` - `stage-analyze.md` - `stage-publish.md` diff --git a/docs/internal/artifacts.md b/docs/internal/artifacts.md index 7bec623..fe32855 100644 --- a/docs/internal/artifacts.md +++ b/docs/internal/artifacts.md @@ -9,6 +9,8 @@ Define canonical artifact IDs, runtime catalog behavior, source resolution rules - `narratio.transcript.polished` -> `transcripts/polished.json` (`polish`) - `narratio.transcript.final` -> `transcripts/final.json` (`normalize`) - `narratio.transcript.final_trimmed` -> `transcripts/final.trimmed.json` (`trim`) +- `narratio.transcript.final_markdown` -> `transcripts/final.md` (`render`) +- `narratio.transcript.final_trimmed_markdown` -> `transcripts/final.trimmed.md` (`render`) - `narratio.bounds.session` -> `artifacts/session_bounds.json` (`trim`) ## Configured and Previous-Session Sources @@ -53,7 +55,8 @@ Previous-session sources (`narratio.previous_session.artifact.*`): Validation by content type: -- transcript built-ins: JSON with top-level `segments` array; +- transcript JSON built-ins: JSON with top-level `segments` array; +- transcript Markdown built-ins: non-empty text file; - bounds built-in: valid JSON; - configured/previous-session artifact files: non-empty text file. diff --git a/docs/internal/stage-analyze.md b/docs/internal/stage-analyze.md index 1181d23..0be9861 100644 --- a/docs/internal/stage-analyze.md +++ b/docs/internal/stage-analyze.md @@ -30,6 +30,7 @@ Supported source families: ## Failure Semantics - required missing configured/previous-session inputs fail. - missing required previous-session source includes prepare rerun guidance. +- missing required `narratio.transcript.final_markdown` or `narratio.transcript.final_trimmed_markdown` inputs includes render rerun guidance. - dependency cycles or unavailable required dependencies fail. - adapter validation failures fail stage. diff --git a/docs/internal/stage-publish.md b/docs/internal/stage-publish.md index 208bd0c..d5e4f19 100644 --- a/docs/internal/stage-publish.md +++ b/docs/internal/stage-publish.md @@ -4,7 +4,7 @@ Upload run/session outputs to object storage and atomically advance remote current state. ## Inputs -- successful prerequisite stages: `prepare`, `transcribe`, `merge`, `polish`, `normalize`, `trim`, `analyze` +- successful prerequisite stages: `prepare`, `transcribe`, `merge`, `polish`, `normalize`, `trim`, `render`, `analyze` - run root `runs/{run_id}/**` - publish output rules (`pipeline.publish.outputs`) - effective publish locks (static + remote merged lock set) diff --git a/docs/internal/stage-render.md b/docs/internal/stage-render.md new file mode 100644 index 0000000..e9ba806 --- /dev/null +++ b/docs/internal/stage-render.md @@ -0,0 +1,29 @@ +# Stage: render + +## Purpose +Render Markdown transcript artifacts from normalized JSON transcripts via Seriatim. + +## Inputs +- `narratio.transcript.final` (`transcripts/final.json`) +- `narratio.transcript.final_trimmed` (`transcripts/final.trimmed.json`) + +## Outputs +- `narratio.transcript.final_markdown` -> `transcripts/final.md` +- `narratio.transcript.final_trimmed_markdown` -> `transcripts/final.trimmed.md` + +## Key Behavior +- uses `pipeline.render` settings (enabled/format/title/booleans). +- resolves inputs manifest-first, then canonical fallback. +- writes run-local outputs first, then materializes canonical session outputs. +- records input provenance, output paths, adapter metadata, logs, and generated config refs. +- skips with stage metadata when `pipeline.render.enabled=false`. + +## Failure Semantics +- missing normalized input fails with normalize rerun guidance. +- missing trimmed input fails with trim rerun guidance. +- adapter/subprocess failure fails stage. +- empty render output files fail validation. + +## Invariants +- only `format: markdown` is supported. +- render stage owns production of built-in Markdown transcript sources. diff --git a/docs/operations.md b/docs/operations.md index a943ad8..49a6296 100644 --- a/docs/operations.md +++ b/docs/operations.md @@ -73,9 +73,10 @@ Canonical stage order: 4. `polish` 5. `normalize` 6. `trim` -7. `analyze` -8. `publish` -9. `notify` +7. `render` +8. `analyze` +9. `publish` +10. `notify` Execution rules: @@ -245,5 +246,6 @@ Rules: - Local and S3 audio modes are mutually exclusive. - Publish requires prerequisite stages through analyze to be succeeded. +- Markdown publish defaults require render outputs (`transcripts/final.md` and `transcripts/final.trimmed.md`). - Restore requires configured object storage and committed remote current state. - Storage-backed commands load filesystem secrets before object-store initialization. diff --git a/examples/pipeline.full.annotated.yml b/examples/pipeline.full.annotated.yml index 0aa8cb6..ba6e5b5 100644 --- a/examples/pipeline.full.annotated.yml +++ b/examples/pipeline.full.annotated.yml @@ -48,6 +48,12 @@ publish: - source: narratio.transcript.final_trimmed dest: transcripts/final.trimmed.json required: true + - source: narratio.transcript.final_markdown + dest: transcripts/final.md + required: true + - source: narratio.transcript.final_trimmed_markdown + dest: transcripts/final.trimmed.md + required: true - source: narratio.artifact.session_recap dest: artifacts/session_recap.md required: true diff --git a/examples/pipeline.production.yml b/examples/pipeline.production.yml index 41701c9..affae6d 100644 --- a/examples/pipeline.production.yml +++ b/examples/pipeline.production.yml @@ -26,6 +26,12 @@ publish: - source: narratio.transcript.final_trimmed dest: transcripts/final.trimmed.json required: true + - source: narratio.transcript.final_markdown + dest: transcripts/final.md + required: true + - source: narratio.transcript.final_trimmed_markdown + dest: transcripts/final.trimmed.md + required: true - source: narratio.artifact.session_recap dest: artifacts/session_recap.md required: true diff --git a/internal/app/operator_helpers_test.go b/internal/app/operator_helpers_test.go index 91d2bd0..db6d8bc 100644 --- a/internal/app/operator_helpers_test.go +++ b/internal/app/operator_helpers_test.go @@ -1002,6 +1002,8 @@ func TestExecutePublishLoadsRemoteLocks(t *testing.T) { _ = stageName } mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "final.trimmed.json"), `{"segments":[]}`) + mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "final.md"), "# final\n") + mustWriteTestFile(t, filepath.Join(workRoot, "transcripts", "final.trimmed.md"), "# final trimmed\n") var stdout bytes.Buffer var stderr bytes.Buffer diff --git a/internal/app/runner_test.go b/internal/app/runner_test.go index 71fc7ce..67e83f9 100644 --- a/internal/app/runner_test.go +++ b/internal/app/runner_test.go @@ -273,7 +273,7 @@ func TestExecuteStagesPublishSkipsRequiredUnselectedConfiguredOutput(t *testing. manifestPath := manifestPathFor(cfg) seed := manifest.New(cfg.Session.SessionID, time.Now().UTC()) seed.Campaign = cfg.Session.Campaign - for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim"} { + for _, stageName := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render"} { seed.MarkStageSucceeded(stageName, time.Now().UTC(), nil) } if err := os.MkdirAll(filepath.Dir(manifestPath), 0o755); err != nil { diff --git a/internal/artifactpolicy/policy_test.go b/internal/artifactpolicy/policy_test.go index a669b95..3d32b99 100644 --- a/internal/artifactpolicy/policy_test.go +++ b/internal/artifactpolicy/policy_test.go @@ -67,6 +67,14 @@ func TestResolvePublishedDestination(t *testing.T) { t.Fatalf("built-in destination = %q, want transcripts/final.trimmed.json", got) } + got, err = ResolvePublishedDestination("narratio.transcript.final_markdown", "", configured) + if err != nil { + t.Fatalf("ResolvePublishedDestination(markdown built-in) error = %v", err) + } + if got != "transcripts/final.md" { + t.Fatalf("markdown built-in destination = %q, want transcripts/final.md", got) + } + got, err = ResolvePublishedDestination("narratio.artifact.session_recap", "", configured) if err != nil { t.Fatalf("ResolvePublishedDestination(configured) error = %v", err) @@ -103,6 +111,7 @@ func TestDescribeScriptoriumInputSource(t *testing.T) { wantErrLike string }{ {name: "built in", source: "narratio.transcript.final_trimmed", wantKind: SourceKindBuiltIn}, + {name: "built in markdown", source: "narratio.transcript.final_markdown", wantKind: SourceKindBuiltIn}, {name: "configured", source: "narratio.artifact.session_recap", wantKind: SourceKindConfiguredArtifact, wantKey: "session_recap"}, {name: "previous", source: "narratio.previous_session.artifact.session_recap", wantKind: SourceKindPreviousArtifact, wantKey: "session_recap", wantPrev: true}, {name: "invalid previous", source: "narratio.previous_session.artifact.", wantErr: ErrInvalidPreviousSessionSource}, diff --git a/internal/config/defaults.go b/internal/config/defaults.go index aa1eab5..0898fda 100644 --- a/internal/config/defaults.go +++ b/internal/config/defaults.go @@ -86,6 +86,8 @@ const ( // Callers should copy this slice before mutating. var DefaultPublishOutputs = []PublishOutputRule{ {Source: artifactmodel.SourceTranscriptFinalTrimmed, Dest: PathTranscriptFinalTrimmed}, + {Source: artifactmodel.SourceTranscriptFinalMarkdown, Dest: artifactmodel.TranscriptPathFinalMarkdown}, + {Source: artifactmodel.SourceTranscriptFinalTrimmedMarkdown, Dest: artifactmodel.TranscriptPathFinalTrimmedMarkdown}, } // DefaultPipelineConfigSearchPaths defines the default search order for diff --git a/internal/config/scriptorium_test.go b/internal/config/scriptorium_test.go index c9acdf6..f8c732e 100644 --- a/internal/config/scriptorium_test.go +++ b/internal/config/scriptorium_test.go @@ -200,6 +200,21 @@ func TestScriptoriumLoadAndValidate(t *testing.T) { transcript: source: narratio.transcript.final_trimmed required: true +`, + }, + { + name: "markdown built in artifact source is accepted", + scriptoriumYAML: `scriptorium: + binary: scriptorium + artifacts: + session_recap: + enabled: true + prompt_id: dnd.session_recap + output_path: artifacts/session_recap.md + inputs: + transcript_markdown: + source: narratio.transcript.final_markdown + required: true `, }, { diff --git a/internal/config/storage_publish_test.go b/internal/config/storage_publish_test.go index 013b79f..a883ecb 100644 --- a/internal/config/storage_publish_test.go +++ b/internal/config/storage_publish_test.go @@ -179,18 +179,29 @@ func TestSpoolAndPublishDefaults(t *testing.T) { if cfg.Pipeline.Publish.UploadRun == nil || !*cfg.Pipeline.Publish.UploadRun { t.Fatalf("publish.upload_run = %#v, want true", cfg.Pipeline.Publish.UploadRun) } - if len(cfg.Pipeline.Publish.Outputs) != 1 { - t.Fatalf("publish.outputs len = %d, want 1 default", len(cfg.Pipeline.Publish.Outputs)) + if len(cfg.Pipeline.Publish.Outputs) != 3 { + t.Fatalf("publish.outputs len = %d, want 3 defaults", len(cfg.Pipeline.Publish.Outputs)) } - item := cfg.Pipeline.Publish.Outputs[0] - if item.Required == nil || !*item.Required { - t.Fatalf("publish.outputs[0].required = %#v, want true", item.Required) + wantBySource := map[string]string{ + "narratio.transcript.final_trimmed": "transcripts/final.trimmed.json", + "narratio.transcript.final_markdown": "transcripts/final.md", + "narratio.transcript.final_trimmed_markdown": "transcripts/final.trimmed.md", } - if item.Source != "narratio.transcript.final_trimmed" { - t.Fatalf("publish.outputs[0].source = %q, want narratio.transcript.final_trimmed", item.Source) + for i, item := range cfg.Pipeline.Publish.Outputs { + if item.Required == nil || !*item.Required { + t.Fatalf("publish.outputs[%d].required = %#v, want true", i, item.Required) + } + wantDest, ok := wantBySource[item.Source] + if !ok { + t.Fatalf("publish.outputs[%d].source = %q, want known default source", i, item.Source) + } + if item.Dest != wantDest { + t.Fatalf("publish.outputs[%d].dest = %q, want %q", i, item.Dest, wantDest) + } + delete(wantBySource, item.Source) } - if item.Dest != "transcripts/final.trimmed.json" { - t.Fatalf("publish.outputs[0].dest = %q, want transcripts/final.trimmed.json", item.Dest) + if len(wantBySource) != 0 { + t.Fatalf("missing default publish outputs for sources: %#v", wantBySource) } } @@ -337,6 +348,15 @@ publish: `, wantDest: "artifacts/session_recap.md", }, + { + name: "markdown built in derives canonical destination", + pipelineYML: testPipelineBaseYAML + ` +publish: + outputs: + - source: narratio.transcript.final_markdown +`, + wantDest: "transcripts/final.md", + }, } for _, tt := range tests { diff --git a/internal/stage/analyze.go b/internal/stage/analyze.go index 6f40658..4725d93 100644 --- a/internal/stage/analyze.go +++ b/internal/stage/analyze.go @@ -670,6 +670,12 @@ func resolveScriptoriumInput( return "", false, nil, fmt.Errorf("normalized transcript input is unavailable; run normalize stage first") case artifacts.ArtifactTranscriptFinalTrimmed: return "", false, nil, fmt.Errorf("trimmed transcript input is unavailable; run trim stage first") + case artifacts.ArtifactTranscriptFinalMarkdown, artifacts.ArtifactTranscriptFinalTrimmedMarkdown: + return "", false, nil, fmt.Errorf( + "rendered markdown transcript input is unavailable for source %q; run narratio run-stage --force render %s", + descriptor.Source.ID, + paths.SessionID, + ) default: return "", false, nil, nil } diff --git a/internal/stage/analyze_test.go b/internal/stage/analyze_test.go index 00f0235..99e11cc 100644 --- a/internal/stage/analyze_test.go +++ b/internal/stage/analyze_test.go @@ -961,6 +961,31 @@ func TestAnalyzeSupportsNormalizedTranscriptSourceWhenConfigured(t *testing.T) { } } +func TestAnalyzeSupportsRenderedMarkdownTranscriptSourceWhenConfigured(t *testing.T) { + env, m, fake := setupAnalyzeEnv(t) + paths := sessionPathsForEnv(env, m.SessionID) + markdownPath := filepath.Join(paths.TranscriptsDir, "final.md") + writeAnalyzeFile(t, markdownPath, "# Session Transcript\n") + + artifact := env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] + artifact.Inputs["transcript"] = config.ScriptoriumInputConfig{ + Source: "narratio.transcript.final_markdown", + Required: true, + } + env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = artifact + + _, err := (analyzeStage{}).Run(context.Background(), env, m) + if err != nil { + t.Fatalf("Run() error = %v", err) + } + if len(fake.RunRequests) != 1 { + t.Fatalf("run requests = %d, want 1", len(fake.RunRequests)) + } + if fake.RunRequests[0].InputPaths["transcript"] != markdownPath { + t.Fatalf("transcript input = %q, want markdown transcript path", fake.RunRequests[0].InputPaths["transcript"]) + } +} + func TestAnalyzeSupportsCanonicalNormalizedTranscriptSourceFromManifestOutput(t *testing.T) { env, m, fake := setupAnalyzeEnv(t) paths := sessionPathsForEnv(env, m.SessionID) @@ -1045,6 +1070,42 @@ func TestAnalyzeFailsWhenNormalizedTranscriptMissing(t *testing.T) { } } +func TestAnalyzeFailsWhenRenderedMarkdownTranscriptMissing(t *testing.T) { + env, m, _ := setupAnalyzeEnv(t) + artifact := env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] + artifact.Inputs["transcript"] = config.ScriptoriumInputConfig{ + Source: "narratio.transcript.final_markdown", + Required: true, + } + env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = artifact + + _, err := (analyzeStage{}).Run(context.Background(), env, m) + if err == nil { + t.Fatal("expected error, got nil") + } + if !strings.Contains(err.Error(), "run narratio run-stage --force render") { + t.Fatalf("error = %q, want render guidance", err.Error()) + } +} + +func TestAnalyzeFailsWhenRenderedTrimmedMarkdownTranscriptMissing(t *testing.T) { + env, m, _ := setupAnalyzeEnv(t) + artifact := env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] + artifact.Inputs["transcript"] = config.ScriptoriumInputConfig{ + Source: "narratio.transcript.final_trimmed_markdown", + Required: true, + } + env.Config.Pipeline.Scriptorium.Artifacts["session_recap"] = artifact + + _, err := (analyzeStage{}).Run(context.Background(), env, m) + if err == nil { + t.Fatal("expected error, got nil") + } + if !strings.Contains(err.Error(), "run narratio run-stage --force render") { + t.Fatalf("error = %q, want render guidance", err.Error()) + } +} + func TestAnalyzeFailsWhenProcessedTranscriptInvalidJSON(t *testing.T) { env, m, _ := setupAnalyzeEnv(t) paths := sessionPathsForEnv(env, m.SessionID) diff --git a/internal/stage/publish.go b/internal/stage/publish.go index 1107c42..3820042 100644 --- a/internal/stage/publish.go +++ b/internal/stage/publish.go @@ -32,6 +32,7 @@ var publishPrerequisiteStages = []string{ "polish", "normalize", "trim", + "render", "analyze", }