Implement render stage runtime and integrate it into pipeline execution

This commit is contained in:
2026-05-25 00:40:06 +00:00
parent 0658f2f642
commit 2fece10215
14 changed files with 593 additions and 32 deletions

View File

@@ -74,6 +74,7 @@ func All() []Stage {
polishStage{},
normalizeStage{},
trimStage{},
renderStage{},
analyzeStage{},
publishStage{},
placeholderStage{name: "notify"},

View File

@@ -101,7 +101,7 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
}
m := manifest.New("2026-05-03", time.Now().UTC())
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "analyze"} {
for _, name := range []string{"prepare", "transcribe", "merge", "polish", "normalize", "trim", "render", "analyze"} {
m.MarkStageSucceeded(name, time.Now().UTC(), nil)
}
m.RunID = "20260516T000000Z-abcdef12"
@@ -194,6 +194,15 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
}
continue
}
if s.Name() == "render" {
if result.Metadata["stage"] != "render" {
t.Fatalf("render metadata = %#v, want stage=render", result.Metadata)
}
if len(result.Outputs) != 2 {
t.Fatalf("render outputs = %#v, want 2 markdown outputs", result.Outputs)
}
continue
}
if s.Name() == "publish" {
if result.Metadata["stage"] != "publish" {
t.Fatalf("publish metadata = %#v, want stage=publish", result.Metadata)

288
internal/stage/render.go Normal file
View File

@@ -0,0 +1,288 @@
package stage
import (
"context"
"errors"
"fmt"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/seriatim"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
type renderStage struct{}
func (renderStage) Name() string { return "render" }
func (renderStage) Declares() IODecl {
return IODecl{
Inputs: []artifacts.Ref{
{Kind: "transcript_final", Category: "transcripts", RelativePath: "transcripts/final.json"},
{Kind: "transcript_final_trimmed", Category: "transcripts", RelativePath: "transcripts/final.trimmed.json"},
},
Outputs: []artifacts.Ref{
{Kind: artifacts.TranscriptOutputKindFinalMarkdown, Category: "transcripts", RelativePath: artifacts.TranscriptPathFinalMarkdown},
{Kind: artifacts.TranscriptOutputKindFinalTrimmedMarkdown, Category: "transcripts", RelativePath: artifacts.TranscriptPathFinalTrimmedMarkdown},
},
}
}
func (renderStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*StageResult, error) {
if env == nil || env.Config == nil {
return nil, fmt.Errorf("render: stage environment config is required")
}
if env.ArtifactStore == nil {
return nil, fmt.Errorf("render: artifact store is required")
}
if env.Config.Pipeline == nil || env.Config.Session == nil {
return nil, fmt.Errorf("render: resolved config must include pipeline and session")
}
if env.Seriatim == nil {
return nil, fmt.Errorf("render: seriatim adapter is required")
}
var sessionID string
if m != nil {
sessionID = strings.TrimSpace(m.SessionID)
}
if sessionID == "" {
sessionID = strings.TrimSpace(env.Config.Session.SessionID)
}
if sessionID == "" {
return nil, fmt.Errorf("render: session id is required")
}
paths := sessionPathsForEnv(env, sessionID)
runLayout, err := resolveRunStageLayout(env, m, paths, sessionID, "render")
if err != nil {
return nil, fmt.Errorf("render: resolve run-stage layout: %w", err)
}
renderCfg := renderConfigOrDefault(env.Config.Pipeline.Render)
enabled := renderCfg.Enabled == nil || *renderCfg.Enabled
format := strings.TrimSpace(renderCfg.Format)
if format == "" {
format = config.DefaultRenderFormat
}
title := resolveRenderTitle(renderCfg, env.Config.Session)
includeTimestamps := renderCfg.IncludeTimestamps == nil || *renderCfg.IncludeTimestamps
includeSegmentIDs := renderCfg.IncludeSegmentIDs
includeMetadata := renderCfg.IncludeMetadata
meta := map[string]any{
"stage": "render",
"render_enabled": enabled,
"format": format,
"title": title,
"include_timestamps": includeTimestamps,
"include_segment_ids": includeSegmentIDs,
"include_metadata": includeMetadata,
"binary": env.Config.Pipeline.Seriatim.Binary,
"timeout": env.Config.Pipeline.Seriatim.Timeout,
}
if !enabled {
meta["skipped"] = true
meta["reason"] = "pipeline.render.enabled is false"
return &StageResult{Metadata: meta}, nil
}
finalInput, err := artifacts.ResolveSessionArtifact(paths, m, artifacts.ArtifactTranscriptFinal)
if err != nil {
return nil, wrapRenderInputResolveError(err, sessionID, artifacts.ArtifactTranscriptFinal, "normalize")
}
finalTrimmedInput, err := artifacts.ResolveSessionArtifact(paths, m, artifacts.ArtifactTranscriptFinalTrimmed)
if err != nil {
return nil, wrapRenderInputResolveError(err, sessionID, artifacts.ArtifactTranscriptFinalTrimmed, "trim")
}
canonicalFinalMarkdownPath, err := resolveScriptoriumOutputPath(paths, artifacts.TranscriptPathFinalMarkdown)
if err != nil {
return nil, fmt.Errorf("render: resolve canonical final markdown output path: %w", err)
}
canonicalFinalTrimmedMarkdownPath, err := resolveScriptoriumOutputPath(paths, artifacts.TranscriptPathFinalTrimmedMarkdown)
if err != nil {
return nil, fmt.Errorf("render: resolve canonical final trimmed markdown output path: %w", err)
}
runFinalMarkdownPath, err := runLocalPathForCanonical(runLayout, paths, canonicalFinalMarkdownPath)
if err != nil {
return nil, fmt.Errorf("render: resolve run-local final markdown output path: %w", err)
}
runFinalTrimmedMarkdownPath, err := runLocalPathForCanonical(runLayout, paths, canonicalFinalTrimmedMarkdownPath)
if err != nil {
return nil, fmt.Errorf("render: resolve run-local final trimmed markdown output path: %w", err)
}
finalStdoutLogPath := filepath.Join(paths.LogsDir, "seriatim.render.final.stdout.log")
finalStderrLogPath := filepath.Join(paths.LogsDir, "seriatim.render.final.stderr.log")
finalGeneratedConfigPath := filepath.Join(paths.ConfigDir, "seriatim.render.final.generated.yml")
finalTrimmedStdoutLogPath := filepath.Join(paths.LogsDir, "seriatim.render.final_trimmed.stdout.log")
finalTrimmedStderrLogPath := filepath.Join(paths.LogsDir, "seriatim.render.final_trimmed.stderr.log")
finalTrimmedGeneratedConfigPath := filepath.Join(paths.ConfigDir, "seriatim.render.final_trimmed.generated.yml")
if runLayout.Enabled {
finalStdoutLogPath = filepath.Join(runLayout.LogsDir, "seriatim.render.final.stdout.log")
finalStderrLogPath = filepath.Join(runLayout.LogsDir, "seriatim.render.final.stderr.log")
finalGeneratedConfigPath = filepath.Join(runLayout.ConfigDir, "seriatim.render.final.generated.yml")
finalTrimmedStdoutLogPath = filepath.Join(runLayout.LogsDir, "seriatim.render.final_trimmed.stdout.log")
finalTrimmedStderrLogPath = filepath.Join(runLayout.LogsDir, "seriatim.render.final_trimmed.stderr.log")
finalTrimmedGeneratedConfigPath = filepath.Join(runLayout.ConfigDir, "seriatim.render.final_trimmed.generated.yml")
}
timeout, err := resolveTrimSeriatimTimeout(env.Config.Pipeline.Seriatim.Timeout)
if err != nil {
return nil, fmt.Errorf("render: resolve seriatim timeout: %w", err)
}
finalReq := seriatim.RenderRequest{
Binary: env.Config.Pipeline.Seriatim.Binary,
InputTranscriptPath: finalInput.Path,
OutputRenderedPath: runFinalMarkdownPath,
Format: format,
Title: title,
IncludeTimestamps: includeTimestamps,
IncludeSegmentIDs: includeSegmentIDs,
IncludeMetadata: includeMetadata,
StdoutLogPath: finalStdoutLogPath,
StderrLogPath: finalStderrLogPath,
GeneratedConfigPath: finalGeneratedConfigPath,
Timeout: timeout,
}
finalRes, err := env.Seriatim.Render(ctx, finalReq)
if err != nil {
return nil, fmt.Errorf("render: seriatim render failed for %q: %w", artifacts.ArtifactTranscriptFinal, err)
}
finalRenderedPath := coalesceString(finalRes.OutputRenderedPath, finalReq.OutputRenderedPath)
if err := requireNonEmptyFile(finalRenderedPath, "final transcript markdown output"); err != nil {
return nil, fmt.Errorf("render: %w", err)
}
finalTrimmedReq := seriatim.RenderRequest{
Binary: env.Config.Pipeline.Seriatim.Binary,
InputTranscriptPath: finalTrimmedInput.Path,
OutputRenderedPath: runFinalTrimmedMarkdownPath,
Format: format,
Title: title,
IncludeTimestamps: includeTimestamps,
IncludeSegmentIDs: includeSegmentIDs,
IncludeMetadata: includeMetadata,
StdoutLogPath: finalTrimmedStdoutLogPath,
StderrLogPath: finalTrimmedStderrLogPath,
GeneratedConfigPath: finalTrimmedGeneratedConfigPath,
Timeout: timeout,
}
finalTrimmedRes, err := env.Seriatim.Render(ctx, finalTrimmedReq)
if err != nil {
return nil, fmt.Errorf("render: seriatim render failed for %q: %w", artifacts.ArtifactTranscriptFinalTrimmed, err)
}
finalTrimmedRenderedPath := coalesceString(finalTrimmedRes.OutputRenderedPath, finalTrimmedReq.OutputRenderedPath)
if err := requireNonEmptyFile(finalTrimmedRenderedPath, "final trimmed transcript markdown output"); err != nil {
return nil, fmt.Errorf("render: %w", err)
}
materializedFinalMarkdown, err := materializeRunLocalOutput(env.ArtifactStore, finalRenderedPath, canonicalFinalMarkdownPath, artifacts.Ref{
Kind: artifacts.TranscriptOutputKindFinalMarkdown,
Category: "transcripts",
SessionID: sessionID,
})
if err != nil {
return nil, fmt.Errorf("render: materialize canonical final markdown output: %w", err)
}
materializedFinalTrimmedMarkdown, err := materializeRunLocalOutput(env.ArtifactStore, finalTrimmedRenderedPath, canonicalFinalTrimmedMarkdownPath, artifacts.Ref{
Kind: artifacts.TranscriptOutputKindFinalTrimmedMarkdown,
Category: "transcripts",
SessionID: sessionID,
})
if err != nil {
return nil, fmt.Errorf("render: materialize canonical final trimmed markdown output: %w", err)
}
meta["final_input_path"] = finalInput.Path
meta["final_input_provenance"] = finalInput.Provenance
meta["final_input_run_id"] = finalInput.ProducerRunID
meta["final_trimmed_input_path"] = finalTrimmedInput.Path
meta["final_trimmed_input_provenance"] = finalTrimmedInput.Provenance
meta["final_trimmed_input_run_id"] = finalTrimmedInput.ProducerRunID
meta["run_final_markdown_path"] = finalRenderedPath
meta["final_markdown_path"] = canonicalFinalMarkdownPath
meta["run_final_trimmed_markdown_path"] = finalTrimmedRenderedPath
meta["final_trimmed_markdown_path"] = canonicalFinalTrimmedMarkdownPath
populateRenderAdapterMetadata(meta, "final_adapter_", finalRes)
populateRenderAdapterMetadata(meta, "final_trimmed_adapter_", finalTrimmedRes)
return &StageResult{
Outputs: []artifacts.Ref{
materializedFinalMarkdown,
materializedFinalTrimmedMarkdown,
},
Logs: dedupeAndSortPaths([]string{
finalStdoutLogPath,
finalStderrLogPath,
finalTrimmedStdoutLogPath,
finalTrimmedStderrLogPath,
}),
GeneratedConfigs: dedupeAndSortPaths([]string{
finalGeneratedConfigPath,
finalTrimmedGeneratedConfigPath,
}),
Metadata: meta,
}, nil
}
func renderConfigOrDefault(cfg *config.RenderConfig) *config.RenderConfig {
if cfg != nil {
return cfg
}
enabled := true
includeTimestamps := true
return &config.RenderConfig{
Enabled: &enabled,
Format: config.DefaultRenderFormat,
IncludeTimestamps: &includeTimestamps,
IncludeSegmentIDs: config.DefaultRenderSegmentIDs,
IncludeMetadata: config.DefaultRenderMetadata,
}
}
func resolveRenderTitle(renderCfg *config.RenderConfig, sessionCfg *config.SessionConfig) string {
if renderCfg != nil && strings.TrimSpace(renderCfg.Title) != "" {
return strings.TrimSpace(renderCfg.Title)
}
if sessionCfg != nil && strings.TrimSpace(sessionCfg.Title) != "" {
return strings.TrimSpace(sessionCfg.Title)
}
return ""
}
func wrapRenderInputResolveError(err error, sessionID, sourceID, guidanceStage string) error {
var notFound *artifacts.SessionArtifactNotFoundError
if errors.As(err, &notFound) {
return fmt.Errorf(
"render: required input %q is unavailable; run narratio run-stage --force %s %s",
sourceID,
guidanceStage,
sessionID,
)
}
return fmt.Errorf("render: resolve %q input: %w", sourceID, err)
}
func populateRenderAdapterMetadata(meta map[string]any, prefix string, result seriatim.RenderResult) {
if meta == nil {
return
}
meta[prefix+"duration_ms"] = result.Duration.Milliseconds()
meta[prefix+"exit_code"] = result.ExitCode
meta[prefix+"invoked_binary"] = result.InvokedBinary
meta[prefix+"format"] = result.Format
meta[prefix+"title"] = result.Title
meta[prefix+"output_path"] = result.OutputRenderedPath
meta[prefix+"generated_config"] = result.GeneratedConfigPath
meta[prefix+"stdout_log_path"] = result.StdoutLogPath
meta[prefix+"stderr_log_path"] = result.StderrLogPath
if result.Metadata != nil {
meta[prefix+"metadata"] = result.Metadata
}
}

View File

@@ -0,0 +1,207 @@
package stage
import (
"context"
"path/filepath"
"strings"
"testing"
"time"
"gitea.maximumdirect.net/eric/narratio/internal/adapters/seriatim"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/config"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
func TestRenderStageDisabledSucceedsWithoutOutputs(t *testing.T) {
env, m, ser := setupRenderEnv(t)
enabled := false
env.Config.Pipeline.Render.Enabled = &enabled
result, err := (renderStage{}).Run(context.Background(), env, m)
if err != nil {
t.Fatalf("render.Run() error = %v", err)
}
if len(result.Outputs) != 0 {
t.Fatalf("outputs = %#v, want empty when disabled", result.Outputs)
}
if len(ser.RenderRequests) != 0 {
t.Fatalf("render requests = %d, want 0 when disabled", len(ser.RenderRequests))
}
if result.Metadata["render_enabled"] != false {
t.Fatalf("metadata render_enabled = %#v, want false", result.Metadata["render_enabled"])
}
}
func TestRenderStageResolvesManifestInputsFirst(t *testing.T) {
env, m, ser := setupRenderEnv(t)
paths := sessionPathsForEnv(env, m.SessionID)
manifestFinal := filepath.Join(paths.ArtifactsDir, "final.from-manifest.json")
manifestTrimmed := filepath.Join(paths.ArtifactsDir, "final.trimmed.from-manifest.json")
writeFile(t, manifestFinal, `{"segments":[{"id":101}]}`)
writeFile(t, manifestTrimmed, `{"segments":[{"id":202}]}`)
writeFile(t, filepath.Join(paths.TranscriptsDir, "final.json"), `{"segments":[{"id":1}]}`)
writeFile(t, filepath.Join(paths.TranscriptsDir, "final.trimmed.json"), `{"segments":[{"id":2}]}`)
m.MarkStageSucceeded("normalize", time.Now().UTC(), []manifest.ArtifactRecord{{Kind: "transcript_final", LocalPath: manifestFinal}})
m.MarkStageSucceeded("trim", time.Now().UTC(), []manifest.ArtifactRecord{{Kind: "transcript_final_trimmed", LocalPath: manifestTrimmed}})
result, err := (renderStage{}).Run(context.Background(), env, m)
if err != nil {
t.Fatalf("render.Run() error = %v", err)
}
if len(ser.RenderRequests) != 2 {
t.Fatalf("render requests = %d, want 2", len(ser.RenderRequests))
}
if ser.RenderRequests[0].InputTranscriptPath != manifestFinal {
t.Fatalf("first input path = %q, want %q", ser.RenderRequests[0].InputTranscriptPath, manifestFinal)
}
if ser.RenderRequests[1].InputTranscriptPath != manifestTrimmed {
t.Fatalf("second input path = %q, want %q", ser.RenderRequests[1].InputTranscriptPath, manifestTrimmed)
}
if result.Metadata["final_input_provenance"] != "manifest.normalize.outputs" {
t.Fatalf("final_input_provenance = %#v, want manifest.normalize.outputs", result.Metadata["final_input_provenance"])
}
if result.Metadata["final_trimmed_input_provenance"] != "manifest.trim.outputs" {
t.Fatalf("final_trimmed_input_provenance = %#v, want manifest.trim.outputs", result.Metadata["final_trimmed_input_provenance"])
}
if len(result.Outputs) != 2 {
t.Fatalf("outputs = %#v, want 2", result.Outputs)
}
if result.Outputs[0].Kind != artifacts.TranscriptOutputKindFinalMarkdown {
t.Fatalf("outputs[0].kind = %q, want %q", result.Outputs[0].Kind, artifacts.TranscriptOutputKindFinalMarkdown)
}
if result.Outputs[1].Kind != artifacts.TranscriptOutputKindFinalTrimmedMarkdown {
t.Fatalf("outputs[1].kind = %q, want %q", result.Outputs[1].Kind, artifacts.TranscriptOutputKindFinalTrimmedMarkdown)
}
}
func TestRenderStageTitleFallsBackToSessionTitle(t *testing.T) {
env, m, ser := setupRenderEnv(t)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "final.json"), `{"segments":[{"id":1}]}`)
writeFile(t, filepath.Join(paths.TranscriptsDir, "final.trimmed.json"), `{"segments":[{"id":2}]}`)
env.Config.Pipeline.Render.Title = ""
env.Config.Session.Title = "Session Title"
_, err := (renderStage{}).Run(context.Background(), env, m)
if err != nil {
t.Fatalf("render.Run() error = %v", err)
}
if len(ser.RenderRequests) != 2 {
t.Fatalf("render requests = %d, want 2", len(ser.RenderRequests))
}
if ser.RenderRequests[0].Title != "Session Title" || ser.RenderRequests[1].Title != "Session Title" {
t.Fatalf("render titles = %q/%q, want Session Title", ser.RenderRequests[0].Title, ser.RenderRequests[1].Title)
}
}
func TestRenderStageFailsWhenFinalInputMissing(t *testing.T) {
env, m, _ := setupRenderEnv(t)
_, err := (renderStage{}).Run(context.Background(), env, m)
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "run narratio run-stage --force normalize") {
t.Fatalf("error = %q, want normalize guidance", err.Error())
}
}
func TestRenderStageFailsWhenFinalTrimmedInputMissing(t *testing.T) {
env, m, _ := setupRenderEnv(t)
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "final.json"), `{"segments":[{"id":1}]}`)
_, err := (renderStage{}).Run(context.Background(), env, m)
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "run narratio run-stage --force trim") {
t.Fatalf("error = %q, want trim guidance", err.Error())
}
}
func TestRenderStageUsesRunLocalOutputsAndMaterializesCanonical(t *testing.T) {
env, m, ser := setupRenderEnv(t)
env.Config.Session.Campaign = "sample-campaign"
m.Campaign = "sample-campaign"
m.RunID = "20260518T010203Z-abcdef12"
paths := sessionPathsForEnv(env, m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "final.json"), `{"segments":[{"id":1}]}`)
writeFile(t, filepath.Join(paths.TranscriptsDir, "final.trimmed.json"), `{"segments":[{"id":2}]}`)
result, err := (renderStage{}).Run(context.Background(), env, m)
if err != nil {
t.Fatalf("render.Run() error = %v", err)
}
if len(ser.RenderRequests) != 2 {
t.Fatalf("render requests = %d, want 2", len(ser.RenderRequests))
}
if !strings.Contains(ser.RenderRequests[0].OutputRenderedPath, filepath.Join("runs", m.RunID, "render", "outputs")) {
t.Fatalf("run output path = %q, want run-local output path", ser.RenderRequests[0].OutputRenderedPath)
}
for _, out := range result.Outputs {
if strings.Contains(out.AbsolutePath, string(filepath.Separator)+"runs"+string(filepath.Separator)) {
t.Fatalf("materialized output path = %q, want canonical session path", out.AbsolutePath)
}
}
if len(result.GeneratedConfigs) != 2 {
t.Fatalf("generated configs = %#v, want two generated config paths", result.GeneratedConfigs)
}
if len(result.Logs) != 4 {
t.Fatalf("logs = %#v, want four log paths", result.Logs)
}
}
func setupRenderEnv(t *testing.T) (*Env, *manifest.Manifest, *seriatim.FakeRunner) {
t.Helper()
workspace := t.TempDir()
cfgDir := t.TempDir()
sessionPath := filepath.Join(cfgDir, "session.yml")
pipelinePath := filepath.Join(cfgDir, "pipeline.yml")
writeFile(t, sessionPath, "session_id: 2026-05-03\n")
writeFile(t, pipelinePath, "workspace:\n root: "+workspace+"\n")
enabled := true
includeTimestamps := true
seriatimReport := false
cfg := &config.Config{
PipelinePath: pipelinePath,
SessionPath: sessionPath,
Pipeline: &config.PipelineConfig{
Workspace: config.WorkspaceConfig{Root: workspace},
Seriatim: config.SeriatimConfig{
Binary: "seriatim",
Timeout: "10m",
OutputSchema: "seriatim-intermediate",
CoalesceGap: func() *float64 { v := 3.0; return &v }(),
Report: &seriatimReport,
},
Render: &config.RenderConfig{
Enabled: &enabled,
Format: "markdown",
Title: "Pipeline Title",
IncludeTimestamps: &includeTimestamps,
IncludeSegmentIDs: false,
IncludeMetadata: false,
},
},
Session: &config.SessionConfig{
SessionID: "2026-05-03",
Campaign: "sample-campaign",
},
}
store := artifacts.NewLocalStore(workspace)
if _, err := store.EnsureLayoutFor("sample-campaign", "2026-05-03"); err != nil {
t.Fatalf("EnsureLayout() error = %v", err)
}
ser := &seriatim.FakeRunner{}
return &Env{
Config: cfg,
ArtifactStore: store,
Seriatim: ser,
}, manifest.New("2026-05-03", time.Now().UTC()), ser
}