Reintroduce normalize stage scaffold

This commit is contained in:
2026-05-10 21:26:53 +00:00
parent 7d61901585
commit 86d2dfa6c5
12 changed files with 272 additions and 94 deletions

122
internal/stage/normalize.go Normal file
View File

@@ -0,0 +1,122 @@
package stage
import (
"context"
"fmt"
"os"
"path/filepath"
"strings"
"gitea.maximumdirect.net/eric/narratio/internal/artifacts"
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
type normalizeStage struct{}
func (normalizeStage) Name() string { return "normalize" }
func (normalizeStage) Declares() IODecl {
return IODecl{
Inputs: []artifacts.Ref{
{Kind: "transcript_processed", Category: "transcripts", RelativePath: "transcripts/processed.json"},
},
Outputs: []artifacts.Ref{
{Kind: "transcript_normalized", Category: "transcripts", RelativePath: "transcripts/normalized.json"},
},
}
}
func (normalizeStage) Run(_ context.Context, env *Env, m *manifest.Manifest) (*StageResult, error) {
if env == nil || env.Config == nil {
return nil, fmt.Errorf("normalize: stage environment config is required")
}
if env.ArtifactStore == nil {
return nil, fmt.Errorf("normalize: artifact store is required")
}
if env.Config.Pipeline == nil || env.Config.Session == nil {
return nil, fmt.Errorf("normalize: resolved config must include pipeline and session")
}
var sessionID string
if m != nil {
sessionID = strings.TrimSpace(m.SessionID)
}
if sessionID == "" {
sessionID = strings.TrimSpace(env.Config.Session.SessionID)
}
if sessionID == "" {
return nil, fmt.Errorf("normalize: session id is required")
}
paths := env.ArtifactStore.SessionPaths(sessionID)
processedPath, processedSource, err := discoverProcessedTranscript(m, paths)
if err != nil {
return nil, fmt.Errorf("normalize: resolve processed transcript: %w", err)
}
if processedPath == "" {
return nil, fmt.Errorf("normalize: processed transcript input is required")
}
if err := validateProcessedTranscriptOutput(processedPath); err != nil {
return nil, fmt.Errorf("normalize: processed transcript %q invalid: %w", processedPath, err)
}
normalizedPath := filepath.Join(paths.TranscriptsDir, "normalized.json")
if err := copyTranscript(env.ArtifactStore, processedPath, normalizedPath); err != nil {
return nil, fmt.Errorf("normalize: copy processed transcript to normalized output: %w", err)
}
if err := validateProcessedTranscriptOutput(normalizedPath); err != nil {
return nil, fmt.Errorf("normalize: normalized transcript %q invalid: %w", normalizedPath, err)
}
return &StageResult{
Outputs: []artifacts.Ref{{
Kind: "transcript_normalized",
Category: "transcripts",
SessionID: sessionID,
AbsolutePath: normalizedPath,
}},
Metadata: map[string]any{
"stage": "normalize",
"placeholder": true,
"normalize_action": "copy_placeholder",
"processed_transcript_path": processedPath,
"processed_transcript_source": processedSource,
"normalized_transcript_path": normalizedPath,
"normalized_transcript_source": "stage.normalize.output",
},
}, nil
}
func discoverNormalizedTranscript(m *manifest.Manifest, paths artifacts.SessionPaths) (string, string, error) {
candidates := []string{}
if m != nil && m.Stages != nil {
if sr := m.Stages["normalize"]; sr != nil {
for _, out := range sr.Outputs {
if out.Kind != "transcript_normalized" {
continue
}
p := strings.TrimSpace(out.LocalPath)
if p == "" {
continue
}
resolved := artifacts.ResolveSessionLocalPathForRead(paths, p)
candidates = append(candidates, filepath.Clean(resolved))
}
}
}
deduped := dedupeAndSortPaths(candidates)
for _, p := range deduped {
if info, err := os.Stat(p); err == nil && !info.IsDir() {
return p, "manifest.normalize.outputs", nil
}
}
fallback := filepath.Join(paths.TranscriptsDir, "normalized.json")
if info, err := os.Stat(fallback); err == nil && !info.IsDir() {
return filepath.Clean(fallback), "fallback.transcripts_dir", nil
}
if len(deduped) > 0 {
return deduped[0], "manifest.normalize.outputs", nil
}
return "", "", nil
}

View File

@@ -92,6 +92,7 @@ func All() []Stage {
transcribeStage{},
mergeStage{},
polishStage{},
normalizeStage{},
trimStage{},
analyzeStage{},
placeholderStage{name: "archive"},

View File

@@ -111,6 +111,18 @@ func TestStagesReturnExpectedMetadata(t *testing.T) {
}
continue
}
if s.Name() == "normalize" {
if result.Metadata["stage"] != "normalize" {
t.Fatalf("normalize metadata = %#v, want stage=normalize", result.Metadata)
}
if result.Metadata["normalize_action"] != "copy_placeholder" {
t.Fatalf("normalize metadata = %#v, want normalize_action=copy_placeholder", result.Metadata)
}
if len(result.Outputs) == 0 || result.Outputs[0].Kind != "transcript_normalized" {
t.Fatalf("normalize outputs = %#v, want transcript_normalized output", result.Outputs)
}
continue
}
if s.Name() == "analyze" {
if result.Metadata["stage"] != "analyze" {
t.Fatalf("analyze metadata = %#v, want stage=analyze", result.Metadata)

View File

@@ -23,7 +23,7 @@ func (trimStage) Name() string { return "trim" }
func (trimStage) Declares() IODecl {
return IODecl{
Inputs: []artifacts.Ref{
{Kind: "transcript_processed", Category: "transcripts", RelativePath: "transcripts/processed.json"},
{Kind: "transcript_normalized", Category: "transcripts", RelativePath: "transcripts/normalized.json"},
},
Outputs: []artifacts.Ref{
{Kind: "transcript_trimmed", Category: "transcripts", RelativePath: "transcripts/trimmed.json"},
@@ -55,15 +55,15 @@ func (trimStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Stag
}
paths := env.ArtifactStore.SessionPaths(sessionID)
processedPath, processedSource, err := discoverProcessedTranscript(m, paths)
normalizedPath, normalizedSource, err := discoverNormalizedTranscript(m, paths)
if err != nil {
return nil, fmt.Errorf("trim: resolve processed transcript: %w", err)
return nil, fmt.Errorf("trim: resolve normalized transcript: %w", err)
}
if processedPath == "" {
return nil, fmt.Errorf("trim: processed transcript input is required")
if normalizedPath == "" {
return nil, fmt.Errorf("trim: normalized transcript input is required")
}
if err := validateProcessedTranscriptOutput(processedPath); err != nil {
return nil, fmt.Errorf("trim: processed transcript %q invalid: %w", processedPath, err)
if err := validateProcessedTranscriptOutput(normalizedPath); err != nil {
return nil, fmt.Errorf("trim: normalized transcript %q invalid: %w", normalizedPath, err)
}
trimCfg := env.Config.Pipeline.Trim
@@ -77,16 +77,16 @@ func (trimStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Stag
logPaths := []string{}
generatedConfigs := []string{}
metadata := map[string]any{
"stage": "trim",
"trim_enabled": enabled,
"processed_transcript_path": processedPath,
"processed_transcript_source": processedSource,
"trimmed_output_path": trimmedPath,
"stage": "trim",
"trim_enabled": enabled,
"normalized_transcript_path": normalizedPath,
"normalized_transcript_source": normalizedSource,
"trimmed_output_path": trimmedPath,
}
if !enabled {
if err := copyTranscript(env.ArtifactStore, processedPath, trimmedPath); err != nil {
return nil, fmt.Errorf("trim: copy processed transcript to trimmed output: %w", err)
if err := copyTranscript(env.ArtifactStore, normalizedPath, trimmedPath); err != nil {
return nil, fmt.Errorf("trim: copy normalized transcript to trimmed output: %w", err)
}
if err := validateProcessedTranscriptOutput(trimmedPath); err != nil {
return nil, fmt.Errorf("trim: copied trimmed transcript %q invalid: %w", trimmedPath, err)
@@ -127,7 +127,7 @@ func (trimStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Stag
}
inputPaths := map[string]string{
boundsCfg.TranscriptInputName: processedPath,
boundsCfg.TranscriptInputName: normalizedPath,
}
vars := map[string]string{}
@@ -136,7 +136,7 @@ func (trimStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Stag
metadata["bounds_output_path"] = boundsOutputPath
metadata["bounds_timeout"] = boundsTimeout.String()
metadata["bounds_input_name"] = boundsCfg.TranscriptInputName
metadata["bounds_input_path"] = processedPath
metadata["bounds_input_path"] = normalizedPath
metadata["bounds_render_debug_enabled"] = boundsCfg.RenderDebug
renderOutputPath := ""
@@ -233,8 +233,8 @@ func (trimStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Stag
if err != nil {
return nil, fmt.Errorf("trim: parse bounds output %q: %w", finalBoundsOutputPath, err)
}
if err := contracts.ValidateSessionBoundsAgainstTranscript(boundsPayload, processedPath); err != nil {
return nil, fmt.Errorf("trim: validate bounds output %q against transcript %q: %w", finalBoundsOutputPath, processedPath, err)
if err := contracts.ValidateSessionBoundsAgainstTranscript(boundsPayload, normalizedPath); err != nil {
return nil, fmt.Errorf("trim: validate bounds output %q against transcript %q: %w", finalBoundsOutputPath, normalizedPath, err)
}
keepSelector, copyUnchanged, err := contracts.BuildSeriatimKeepSelector(boundsPayload)
@@ -272,8 +272,8 @@ func (trimStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Stag
}
if copyUnchanged {
if err := copyTranscript(env.ArtifactStore, processedPath, trimmedPath); err != nil {
return nil, fmt.Errorf("trim: copy processed transcript to trimmed output: %w", err)
if err := copyTranscript(env.ArtifactStore, normalizedPath, trimmedPath); err != nil {
return nil, fmt.Errorf("trim: copy normalized transcript to trimmed output: %w", err)
}
} else {
trimStdoutLogPath := filepath.Join(paths.LogsDir, "seriatim.trim.stdout.log")
@@ -285,7 +285,7 @@ func (trimStage) Run(ctx context.Context, env *Env, m *manifest.Manifest) (*Stag
}
trimReq := seriatim.TrimRequest{
Binary: env.Config.Pipeline.Seriatim.Binary,
InputTranscriptPath: processedPath,
InputTranscriptPath: normalizedPath,
OutputTrimmedPath: trimmedPath,
KeepSelector: keepSelector,
StdoutLogPath: trimStdoutLogPath,

View File

@@ -16,13 +16,13 @@ import (
"gitea.maximumdirect.net/eric/narratio/internal/manifest"
)
func TestTrimStageConsumesProcessedAndProducesTrimmedTranscript(t *testing.T) {
func TestTrimStageConsumesNormalizedAndProducesTrimmedTranscript(t *testing.T) {
env, m, scr, ser := setupTrimEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
processed := filepath.Join(paths.TranscriptsDir, "processed.json")
writeFile(t, processed, `{"segments":[{"id":10},{"id":868}]}`)
m.MarkStageSucceeded("polish", time.Now().UTC(), []manifest.ArtifactRecord{
{Kind: "transcript_processed", LocalPath: processed},
normalized := filepath.Join(paths.TranscriptsDir, "normalized.json")
writeFile(t, normalized, `{"segments":[{"id":10},{"id":868}]}`)
m.MarkStageSucceeded("normalize", time.Now().UTC(), []manifest.ArtifactRecord{
{Kind: "transcript_normalized", LocalPath: normalized},
})
scr.BoundsBody = `{"confidence":"high","trim_action":"trim","start_segment_id":10,"end_segment_id":868,"warnings":[]}`
@@ -38,8 +38,8 @@ func TestTrimStageConsumesProcessedAndProducesTrimmedTranscript(t *testing.T) {
t.Fatalf("seriatim trim requests = %d, want 1", len(ser.TrimRequests))
}
trimReq := ser.TrimRequests[0]
if trimReq.InputTranscriptPath != processed {
t.Fatalf("trim input = %q, want %q", trimReq.InputTranscriptPath, processed)
if trimReq.InputTranscriptPath != normalized {
t.Fatalf("trim input = %q, want %q", trimReq.InputTranscriptPath, normalized)
}
if trimReq.KeepSelector != "10-868" {
t.Fatalf("keep selector = %q, want %q", trimReq.KeepSelector, "10-868")
@@ -71,8 +71,8 @@ func TestTrimStageConsumesProcessedAndProducesTrimmedTranscript(t *testing.T) {
func TestTrimStageUsesConfiguredScriptoriumInputName(t *testing.T) {
env, m, scr, _ := setupTrimEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
processed := filepath.Join(paths.TranscriptsDir, "processed.json")
writeFile(t, processed, `{"segments":[{"id":10},{"id":11}]}`)
normalized := filepath.Join(paths.TranscriptsDir, "normalized.json")
writeFile(t, normalized, `{"segments":[{"id":10},{"id":11}]}`)
scr.BoundsBody = `{"trim_action":"trim","start_segment_id":10,"end_segment_id":11}`
cfg := env.Config.Pipeline.Trim
@@ -87,15 +87,15 @@ func TestTrimStageUsesConfiguredScriptoriumInputName(t *testing.T) {
t.Fatalf("scriptorium run requests = %d, want 1", len(scr.RunRequests))
}
req := scr.RunRequests[0]
if req.InputPaths["transcript_body"] != processed {
t.Fatalf("configured input path = %q, want %q", req.InputPaths["transcript_body"], processed)
if req.InputPaths["transcript_body"] != normalized {
t.Fatalf("configured input path = %q, want %q", req.InputPaths["transcript_body"], normalized)
}
}
func TestTrimStageRecordsLogAndGeneratedConfigRefs(t *testing.T) {
env, m, scr, _ := setupTrimEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":1},{"id":2}]}`)
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
scr.BoundsBody = `{"trim_action":"trim","start_segment_id":1,"end_segment_id":2}`
result, err := (trimStage{}).Run(context.Background(), env, m)
@@ -115,7 +115,7 @@ func TestTrimStageRecordsLogAndGeneratedConfigRefs(t *testing.T) {
func TestTrimStageRenderDebugDiagnosticsAreNotStageOutputs(t *testing.T) {
env, m, scr, _ := setupTrimEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":1},{"id":2}]}`)
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
scr.BoundsBody = `{"trim_action":"trim","start_segment_id":1,"end_segment_id":2}`
cfg := *env.Config.Pipeline.Trim
@@ -139,21 +139,21 @@ func TestTrimStageRenderDebugDiagnosticsAreNotStageOutputs(t *testing.T) {
}
}
func TestTrimStageFailsWhenProcessedTranscriptMissing(t *testing.T) {
func TestTrimStageFailsWhenNormalizedTranscriptMissing(t *testing.T) {
env, m, _, _ := setupTrimEnv(t)
_, err := (trimStage{}).Run(context.Background(), env, m)
if err == nil {
t.Fatal("expected error, got nil")
}
if !strings.Contains(err.Error(), "processed transcript input is required") {
if !strings.Contains(err.Error(), "normalized transcript input is required") {
t.Fatalf("error = %q", err.Error())
}
}
func TestTrimStageFailsWhenProcessedTranscriptInvalidJSON(t *testing.T) {
func TestTrimStageFailsWhenNormalizedTranscriptInvalidJSON(t *testing.T) {
env, m, _, _ := setupTrimEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), "not-json")
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), "not-json")
_, err := (trimStage{}).Run(context.Background(), env, m)
if err == nil {
t.Fatal("expected error, got nil")
@@ -163,10 +163,10 @@ func TestTrimStageFailsWhenProcessedTranscriptInvalidJSON(t *testing.T) {
}
}
func TestTrimStageFailsWhenProcessedTranscriptMissingSegmentsArray(t *testing.T) {
func TestTrimStageFailsWhenNormalizedTranscriptMissingSegmentsArray(t *testing.T) {
env, m, _, _ := setupTrimEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"schema":"audita.processed.v1"}`)
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), `{"schema":"audita.processed.v1"}`)
_, err := (trimStage{}).Run(context.Background(), env, m)
if err == nil {
t.Fatal("expected error, got nil")
@@ -179,7 +179,7 @@ func TestTrimStageFailsWhenProcessedTranscriptMissingSegmentsArray(t *testing.T)
func TestTrimStageFailsWhenBoundsOutputInvalidJSON(t *testing.T) {
env, m, scr, _ := setupTrimEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":1},{"id":2}]}`)
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
scr.BoundsBody = "not-json"
_, err := (trimStage{}).Run(context.Background(), env, m)
if err == nil {
@@ -193,7 +193,7 @@ func TestTrimStageFailsWhenBoundsOutputInvalidJSON(t *testing.T) {
func TestTrimStageFailsWhenBoundsRangeIsDescending(t *testing.T) {
env, m, scr, _ := setupTrimEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":1},{"id":2}]}`)
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
scr.BoundsBody = `{"trim_action":"trim","start_segment_id":2,"end_segment_id":1}`
_, err := (trimStage{}).Run(context.Background(), env, m)
if err == nil {
@@ -207,7 +207,7 @@ func TestTrimStageFailsWhenBoundsRangeIsDescending(t *testing.T) {
func TestTrimStageFailsWhenBoundsIDsMissingFromTranscript(t *testing.T) {
env, m, scr, _ := setupTrimEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":20},{"id":21}]}`)
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), `{"segments":[{"id":20},{"id":21}]}`)
scr.BoundsBody = `{"trim_action":"trim","start_segment_id":10,"end_segment_id":21}`
_, err := (trimStage{}).Run(context.Background(), env, m)
if err == nil {
@@ -221,7 +221,7 @@ func TestTrimStageFailsWhenBoundsIDsMissingFromTranscript(t *testing.T) {
func TestTrimStageFailsWhenScriptoriumAdapterFails(t *testing.T) {
env, m, scr, _ := setupTrimEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":1},{"id":2}]}`)
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
scr.RunErr = errors.New("bounds failed")
_, err := (trimStage{}).Run(context.Background(), env, m)
if err == nil {
@@ -235,7 +235,7 @@ func TestTrimStageFailsWhenScriptoriumAdapterFails(t *testing.T) {
func TestTrimStageFailsWhenSeriatimTrimAdapterFails(t *testing.T) {
env, m, scr, ser := setupTrimEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
writeFile(t, filepath.Join(paths.TranscriptsDir, "processed.json"), `{"segments":[{"id":1},{"id":2}]}`)
writeFile(t, filepath.Join(paths.TranscriptsDir, "normalized.json"), `{"segments":[{"id":1},{"id":2}]}`)
scr.BoundsBody = `{"trim_action":"trim","start_segment_id":1,"end_segment_id":2}`
ser.TrimErr = errors.New("trim failed")
_, err := (trimStage{}).Run(context.Background(), env, m)
@@ -247,12 +247,12 @@ func TestTrimStageFailsWhenSeriatimTrimAdapterFails(t *testing.T) {
}
}
func TestTrimStageDisabledCopiesProcessedTranscript(t *testing.T) {
func TestTrimStageDisabledCopiesNormalizedTranscript(t *testing.T) {
env, m, scr, ser := setupTrimEnv(t)
paths := env.ArtifactStore.SessionPaths(m.SessionID)
processed := filepath.Join(paths.TranscriptsDir, "processed.json")
processedBody := `{"segments":[{"id":1,"text":"alpha"},{"id":2,"text":"beta"}]}`
writeFile(t, processed, processedBody)
normalized := filepath.Join(paths.TranscriptsDir, "normalized.json")
normalizedBody := `{"segments":[{"id":1,"text":"alpha"},{"id":2,"text":"beta"}]}`
writeFile(t, normalized, normalizedBody)
disabled := *env.Config.Pipeline.Trim
disabled.Enabled = false
@@ -273,8 +273,8 @@ func TestTrimStageDisabledCopiesProcessedTranscript(t *testing.T) {
if err != nil {
t.Fatalf("read trimmed transcript: %v", err)
}
if string(data) != processedBody {
t.Fatalf("trimmed body = %q, want copied processed body", string(data))
if string(data) != normalizedBody {
t.Fatalf("trimmed body = %q, want copied normalized body", string(data))
}
}