From e2ae7f77d80e3ed7e58b20eb75e04ef762e8f8c2 Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Mon, 11 May 2026 00:13:26 +0000 Subject: [PATCH] Run normalization in process command --- internal/cli/run.go | 21 +++- internal/cli/run_test.go | 213 ++++++++++++++++++++++++++++++++++----- 2 files changed, 206 insertions(+), 28 deletions(-) diff --git a/internal/cli/run.go b/internal/cli/run.go index 6b27e86..047fa9e 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -10,6 +10,7 @@ import ( "gitea.maximumdirect.net/eric/audita/internal/core/config" "gitea.maximumdirect.net/eric/audita/internal/core/io" + "gitea.maximumdirect.net/eric/audita/internal/core/normalization" "gitea.maximumdirect.net/eric/audita/internal/core/reporting" "gitea.maximumdirect.net/eric/audita/internal/core/schema" ) @@ -45,8 +46,8 @@ var processRunner = func(inv processInvocation, stdout io.Writer) error { return fmt.Errorf("invalid glossary: %w", err) } - // Convert to canonical transcript format for output - outputTranscript := &schema.Transcript{ + // Convert to canonical transcript format for normalization + normalizedTranscript := &schema.Transcript{ Segments: make([]schema.Segment, len(transcript.Segments)), } for i, s := range transcript.Segments { @@ -54,7 +55,7 @@ var processRunner = func(inv processInvocation, stdout io.Writer) error { if s.ID != nil { id = *s.ID } - outputTranscript.Segments[i] = schema.Segment{ + normalizedTranscript.Segments[i] = schema.Segment{ ID: id, Speaker: s.Speaker, Start: s.Start, @@ -64,8 +65,18 @@ var processRunner = func(inv processInvocation, stdout io.Writer) error { } } - // Serialize output transcript to JSON - outputBytes, err := schema.TranscriptToJSON(outputTranscript) + // Apply deterministic normalization + normalizer := normalization.NewNormalizer(normalization.NormalizationConfig{ + MaxSegmentGap: inv.Config.Normalization.MaxSegmentGap, + EllipsisGap: inv.Config.Normalization.EllipsisGap, + MaxSegmentDuration: inv.Config.Normalization.MaxSegmentDuration, + MaxSegmentTokens: inv.Config.Normalization.MaxSegmentTokens, + }) + + normalizedTranscript, _ = normalizer.Normalize(normalizedTranscript) + + // Serialize normalized transcript to JSON + outputBytes, err := schema.TranscriptToJSON(normalizedTranscript) if err != nil { return fmt.Errorf("failed to serialize transcript: %w", err) } diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index f521fb6..9deae50 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -507,35 +507,202 @@ func TestRunProcessEmptyStdoutWithOutput(t *testing.T) { } } -func TestRunProcessReportJSONOnParseFailure(t *testing.T) { +func TestRunProcessNormalizationMergeInOutput(t *testing.T) { var stdout bytes.Buffer var stderr bytes.Buffer - reportPath := filepath.Join(t.TempDir(), "report.json") - exitCode := Run([]string{ - "process", - fixturePath("transcript_empty_speaker.json"), - "--glossary", - fixturePath("tiny_glossary.yaml"), - "--report-json", - reportPath, - }, &stdout, &stderr) - if exitCode == 0 { - t.Fatalf("expected nonzero exit code for parse failure") - } - if stdout.Len() != 0 { - t.Fatalf("expected empty stdout on failure, got %q", stdout.String()) + // Create a test transcript with segments that should be merged + transcriptJSON := `[ + {"id": 1, "speaker": "Alice", "start": 0.0, "end": 1.0, "text": "Hello"}, + {"id": 2, "speaker": "Alice", "start": 1.5, "end": 2.5, "text": "world"} + ]` + + transcriptPath := filepath.Join(t.TempDir(), "transcript.json") + glossaryPath := fixturePath("tiny_glossary.yaml") + outputPath := filepath.Join(t.TempDir(), "corrected.json") + + if err := os.WriteFile(transcriptPath, []byte(transcriptJSON), 0644); err != nil { + t.Fatalf("failed to create test transcript: %v", err) } - // Report should still be written on parse failure - report := readProcessReport(t, reportPath) - if report.Status != "failed" { - t.Fatalf("expected failed report status, got %q", report.Status) + exitCode := Run([]string{"process", transcriptPath, "--glossary", glossaryPath, "--output", outputPath}, &stdout, &stderr) + if exitCode != 0 { + t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String()) } - if report.ErrorMessage == "" { - t.Fatalf("expected error message in failed report") + if stdout.Len() != 0 { + t.Fatalf("expected empty stdout when --output is used, got %q", stdout.String()) } - if !strings.Contains(report.ErrorMessage, "invalid transcript:") { - t.Fatalf("expected invalid transcript error in report, got %q", report.ErrorMessage) + if stderr.Len() != 0 { + t.Fatalf("expected empty stderr on success, got %q", stderr.String()) + } + + // Read and parse the output + outputBytes := readFile(t, outputPath) + var outputTranscript schema.Transcript + if err := schema.ParseTranscriptJSON(outputBytes); err != nil { + t.Fatalf("failed to parse output transcript: %v (output: %s)", err, string(outputBytes)) + } + + // Should have merged the two segments into one + if len(outputTranscript.Segments) != 1 { + t.Fatalf("expected 1 merged segment, got %d segments", len(outputTranscript.Segments)) + } + + // Should have sequential ID starting at 1 + if outputTranscript.Segments[0].ID != 1 { + t.Fatalf("expected segment ID 1, got %d", outputTranscript.Segments[0].ID) + } + + // Should contain merged text + if !strings.Contains(outputTranscript.Segments[0].Text, "Hello") || !strings.Contains(outputTranscript.Segments[0].Text, "world") { + t.Fatalf("expected merged text to contain both parts, got %q", outputTranscript.Segments[0].Text) + } +} + +func TestRunProcessNormalizedOutputToStdout(t *testing.T) { + var stdout bytes.Buffer + var stderr bytes.Buffer + + transcriptJSON := `[ + {"id": 1, "speaker": "Alice", "start": 0.0, "end": 1.0, "text": "Hello"}, + {"id": 2, "speaker": "Alice", "start": 1.5, "end": 2.5, "text": "world"} + ]` + + transcriptPath := filepath.Join(t.TempDir(), "transcript.json") + glossaryPath := fixturePath("tiny_glossary.yaml") + + if err := os.WriteFile(transcriptPath, []byte(transcriptJSON), 0644); err != nil { + t.Fatalf("failed to create test transcript: %v", err) + } + + exitCode := Run([]string{"process", transcriptPath, "--glossary", glossaryPath}, &stdout, &stderr) + if exitCode != 0 { + t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String()) + } + if stderr.Len() != 0 { + t.Fatalf("expected empty stderr on success, got %q", stderr.String()) + } + + // Should have normalized output in stdout + if stdout.Len() == 0 { + t.Fatalf("expected normalized transcript in stdout, got empty stdout") + } + + // Should be valid JSON + if !json.Valid(stdout.Bytes()) { + t.Fatalf("expected valid JSON in stdout, got %q", stdout.String()) + } + + // Parse and verify it's normalized (merged) + var outputTranscript schema.Transcript + if err := schema.ParseTranscriptJSON(stdout.Bytes()); err != nil { + t.Fatalf("failed to parse stdout transcript: %v", err) + } + + if len(outputTranscript.Segments) != 1 { + t.Fatalf("expected 1 merged segment in stdout, got %d", len(outputTranscript.Segments)) + } +} + +func TestRunProcessSequentialIDsInOutput(t *testing.T) { + var stdout bytes.Buffer + var stderr bytes.Buffer + + transcriptJSON := `[ + {"id": 5, "speaker": "Alice", "start": 0.0, "end": 1.0, "text": "Hello"}, + {"id": 10, "speaker": "Bob", "start": 2.0, "end": 3.0, "text": "Hi"} + ]` + + transcriptPath := filepath.Join(t.TempDir(), "transcript.json") + glossaryPath := fixturePath("tiny_glossary.yaml") + + if err := os.WriteFile(transcriptPath, []byte(transcriptJSON), 0644); err != nil { + t.Fatalf("failed to create test transcript: %v", err) + } + + exitCode := Run([]string{"process", transcriptPath, "--glossary", glossaryPath}, &stdout, &stderr) + if exitCode != 0 { + t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String()) + } + + // Parse output and verify sequential IDs + var outputTranscript schema.Transcript + if err := schema.ParseTranscriptJSON(stdout.Bytes()); err != nil { + t.Fatalf("failed to parse output transcript: %v", err) + } + + // Should have sequential IDs starting at 1 + for i, segment := range outputTranscript.Segments { + expectedID := i + 1 + if segment.ID != expectedID { + t.Fatalf("expected segment %d to have ID %d, got %d", i, expectedID, segment.ID) + } + } +} + +func TestRunProcessNoStdoutWithOutput(t *testing.T) { + var stdout bytes.Buffer + var stderr bytes.Buffer + + transcriptJSON := `[ + {"id": 1, "speaker": "Alice", "start": 0.0, "end": 1.0, "text": "Hello"} + ]` + + transcriptPath := filepath.Join(t.TempDir(), "transcript.json") + glossaryPath := fixturePath("tiny_glossary.yaml") + outputPath := filepath.Join(t.TempDir(), "corrected.json") + + if err := os.WriteFile(transcriptPath, []byte(transcriptJSON), 0644); err != nil { + t.Fatalf("failed to create test transcript: %v", err) + } + + exitCode := Run([]string{"process", transcriptPath, "--glossary", glossaryPath, "--output", outputPath}, &stdout, &stderr) + if exitCode != 0 { + t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String()) + } + if stdout.Len() != 0 { + t.Fatalf("expected empty stdout when --output is used, got %q", stdout.String()) + } + if stderr.Len() != 0 { + t.Fatalf("expected empty stderr on success, got %q", stderr.String()) + } + + // Verify output file exists and is valid + if _, err := os.Stat(outputPath); err != nil { + t.Fatalf("expected output file to be created: %v", err) + } +} + +func TestRunProcessNormalizationConfigOverride(t *testing.T) { + var stdout bytes.Buffer + var stderr bytes.Buffer + + // Create segments that would normally merge but set very small max gap + transcriptJSON := `[ + {"id": 1, "speaker": "Alice", "start": 0.0, "end": 1.0, "text": "Hello"}, + {"id": 2, "speaker": "Alice", "start": 1.5, "end": 2.5, "text": "world"} + ]` + + transcriptPath := filepath.Join(t.TempDir(), "transcript.json") + glossaryPath := fixturePath("tiny_glossary.yaml") + + if err := os.WriteFile(transcriptPath, []byte(transcriptJSON), 0644); err != nil { + t.Fatalf("failed to create test transcript: %v", err) + } + + // Set max segment gap to 0.1 (smaller than the 0.5 gap between segments) + exitCode := Run([]string{"process", transcriptPath, "--glossary", glossaryPath, "--normalize-max-segment-gap", "0.1"}, &stdout, &stderr) + if exitCode != 0 { + t.Fatalf("expected exit code 0, got %d with stderr %q", exitCode, stderr.String()) + } + + // Should NOT merge due to small max gap setting + var outputTranscript schema.Transcript + if err := schema.ParseTranscriptJSON(stdout.Bytes()); err != nil { + t.Fatalf("failed to parse output transcript: %v", err) + } + + if len(outputTranscript.Segments) != 2 { + t.Fatalf("expected 2 unmerged segments due to small max gap, got %d", len(outputTranscript.Segments)) } }