eric/seriatim
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Updated the normalize command to correct common errors in WhisperX-generated input transcripts
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
Details

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-16 23:05:42 -05:00
parent 6dbb7ab17e
commit b20438acf0
7 changed files with 357 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

134
internal/normalize/parse_test.go
View File

@@ -92,34 +92,70 @@ func TestParseReaderTopLevelScalarShapesFail(t *testing.T) {
}
}
func TestParseReaderMissingStartFails(t *testing.T) {
_, err := ParseReader(strings.NewReader(`[{"end":2,"speaker":"A","text":"t"}]`))
assertContains(t, err, `missing required field "start"`)
func TestParseReaderMissingStartUsesEndValue(t *testing.T) {
parsed, err := ParseReader(strings.NewReader(`[{"end":2,"speaker":"A","text":"t"}]`))
if err != nil {
t.Fatalf("parse failed: %v", err)
}
if len(parsed.Segments) != 1 {
t.Fatalf("segment count = %d, want 1", len(parsed.Segments))
}
if parsed.Segments[0].Start != 2 || parsed.Segments[0].End != 2 {
t.Fatalf("segment timing = %v..%v, want 2..2", parsed.Segments[0].Start, parsed.Segments[0].End)
}
}
func TestParseReaderMissingEndFails(t *testing.T) {
_, err := ParseReader(strings.NewReader(`[{"start":1,"speaker":"A","text":"t"}]`))
assertContains(t, err, `missing required field "end"`)
func TestParseReaderMissingEndUsesStartValue(t *testing.T) {
parsed, err := ParseReader(strings.NewReader(`[{"start":1,"speaker":"A","text":"t"}]`))
if err != nil {
t.Fatalf("parse failed: %v", err)
}
if len(parsed.Segments) != 1 {
t.Fatalf("segment count = %d, want 1", len(parsed.Segments))
}
if parsed.Segments[0].Start != 1 || parsed.Segments[0].End != 1 {
t.Fatalf("segment timing = %v..%v, want 1..1", parsed.Segments[0].Start, parsed.Segments[0].End)
}
}
func TestParseReaderMissingSpeakerFails(t *testing.T) {
_, err := ParseReader(strings.NewReader(`[{"start":1,"end":2,"text":"t"}]`))
assertContains(t, err, `missing required field "speaker"`)
func TestParseReaderMissingSpeakerUsesUnknownPlaceholder(t *testing.T) {
parsed, err := ParseReader(strings.NewReader(`[{"start":1,"end":2,"text":"t"}]`))
if err != nil {
t.Fatalf("parse failed: %v", err)
}
if parsed.Segments[0].Speaker != "Unknown_Speaker" {
t.Fatalf("speaker = %q, want Unknown_Speaker", parsed.Segments[0].Speaker)
}
}
func TestParseReaderEmptySpeakerFails(t *testing.T) {
_, err := ParseReader(strings.NewReader(`[{"start":1,"end":2,"speaker":" ","text":"t"}]`))
assertContains(t, err, `speaker must be non-empty`)
func TestParseReaderEmptySpeakerUsesUnknownPlaceholder(t *testing.T) {
parsed, err := ParseReader(strings.NewReader(`[{"start":1,"end":2,"speaker":" ","text":"t"}]`))
if err != nil {
t.Fatalf("parse failed: %v", err)
}
if parsed.Segments[0].Speaker != "Unknown_Speaker" {
t.Fatalf("speaker = %q, want Unknown_Speaker", parsed.Segments[0].Speaker)
}
}
func TestParseReaderMissingTextFails(t *testing.T) {
_, err := ParseReader(strings.NewReader(`[{"start":1,"end":2,"speaker":"A"}]`))
assertContains(t, err, `missing required field "text"`)
func TestParseReaderMissingTextDropsSegment(t *testing.T) {
parsed, err := ParseReader(strings.NewReader(`[{"start":1,"end":2,"speaker":"A"}]`))
if err != nil {
t.Fatalf("parse failed: %v", err)
}
if len(parsed.Segments) != 0 {
t.Fatalf("segment count = %d, want 0", len(parsed.Segments))
}
}
func TestParseReaderEndBeforeStartFails(t *testing.T) {
_, err := ParseReader(strings.NewReader(`[{"start":3,"end":2,"speaker":"A","text":"t"}]`))
assertContains(t, err, "before start")
func TestParseReaderEndBeforeStartSwapsValues(t *testing.T) {
parsed, err := ParseReader(strings.NewReader(`[{"start":3,"end":2,"speaker":"A","text":"t"}]`))
if err != nil {
t.Fatalf("parse failed: %v", err)
}
if parsed.Segments[0].Start != 2 || parsed.Segments[0].End != 3 {
t.Fatalf("segment timing = %v..%v, want 2..3", parsed.Segments[0].Start, parsed.Segments[0].End)
}
}
func TestParseReaderNegativeStartFails(t *testing.T) {
@@ -153,6 +189,68 @@ func TestParseReaderCategoriesPreservedWhenValid(t *testing.T) {
}
}
func TestParseReaderMissingBothTimesMiddleUsesNeighborMidpoint(t *testing.T) {
parsed, err := ParseReader(strings.NewReader(`[
{"start":1,"end":2,"speaker":"A","text":"left"},
{"speaker":"B","text":"middle"},
{"start":6,"end":7,"speaker":"C","text":"right"}
]`))
if err != nil {
t.Fatalf("parse failed: %v", err)
}
if parsed.Segments[1].Start != 4 || parsed.Segments[1].End != 4 {
t.Fatalf("middle timing = %v..%v, want 4..4", parsed.Segments[1].Start, parsed.Segments[1].End)
}
}
func TestParseReaderMissingBothTimesEdgeFallbacks(t *testing.T) {
parsedFirst, err := ParseReader(strings.NewReader(`[
{"speaker":"A","text":"first"},
{"start":5,"end":6,"speaker":"B","text":"second"}
]`))
if err != nil {
t.Fatalf("parse first failed: %v", err)
}
if parsedFirst.Segments[0].Start != 5 || parsedFirst.Segments[0].End != 5 {
t.Fatalf("first timing = %v..%v, want 5..5", parsedFirst.Segments[0].Start, parsedFirst.Segments[0].End)
}
parsedLast, err := ParseReader(strings.NewReader(`[
{"start":1,"end":2,"speaker":"A","text":"first"},
{"speaker":"B","text":"last"}
]`))
if err != nil {
t.Fatalf("parse last failed: %v", err)
}
if parsedLast.Segments[1].Start != 2 || parsedLast.Segments[1].End != 2 {
t.Fatalf("last timing = %v..%v, want 2..2", parsedLast.Segments[1].Start, parsedLast.Segments[1].End)
}
}
func TestParseReaderMissingBothTimesSingleSegmentUsesZero(t *testing.T) {
parsed, err := ParseReader(strings.NewReader(`[{"speaker":"A","text":"only"}]`))
if err != nil {
t.Fatalf("parse failed: %v", err)
}
if parsed.Segments[0].Start != 0 || parsed.Segments[0].End != 0 {
t.Fatalf("timing = %v..%v, want 0..0", parsed.Segments[0].Start, parsed.Segments[0].End)
}
}
func TestParseReaderEmptyOrWhitespaceTextDropped(t *testing.T) {
parsed, err := ParseReader(strings.NewReader(`[
{"start":1,"end":2,"speaker":"A","text":"ok"},
{"start":2,"end":3,"speaker":"A","text":""},
{"start":3,"end":4,"speaker":"A","text":" "}
]`))
if err != nil {
t.Fatalf("parse failed: %v", err)
}
if len(parsed.Segments) != 1 {
t.Fatalf("segment count = %d, want 1", len(parsed.Segments))
}
}
func TestParseReaderOriginalInputIndexPreserved(t *testing.T) {
input := `[
{"start":1,"end":2,"speaker":"A","text":"one"},