eric/seriatim
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 7 Wiki Activity

Updated the normalize command to correct common errors in WhisperX-generated input transcripts
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
Details

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-16 23:05:42 -05:00
parent 6dbb7ab17e
commit b20438acf0
7 changed files with 357 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
internal/normalize/normalize.go
View File

@@ -19,11 +19,16 @@ type normalizeAudit struct {
OutputFile string `json:"output_file"`
InputShape string `json:"input_shape"`
InputSegmentCount int `json:"input_segment_count"`
OutputSegmentCount int `json:"output_segment_count"`
OutputSchema string `json:"output_schema"`
OutputModules []string `json:"output_modules"`
IDsReassigned bool `json:"ids_reassigned"`
SortingChangedInput bool `json:"sorting_changed_input_order"`
SegmentsWithCategories int `json:"segments_with_categories"`
TimingFieldsRepaired int `json:"timing_fields_repaired"`
TimingOrderSwapped int `json:"timing_order_swapped"`
SpeakerFilled int `json:"speaker_filled"`
SegmentsDroppedText int `json:"segments_dropped_text"`
}
// Run executes artifact-level normalization.
@@ -52,12 +57,17 @@ func Run(ctx context.Context, cfg config.NormalizeConfig) error {
InputFile: cfg.InputFile,
OutputFile: cfg.OutputFile,
InputShape: string(parsed.Shape),
InputSegmentCount: len(parsed.Segments),
InputSegmentCount: parsed.InputSegmentCount,
OutputSegmentCount: built.OutputSegmentCount,
OutputSchema: cfg.OutputSchema,
OutputModules: append([]string(nil), cfg.OutputModules...),
IDsReassigned: built.IDsReassigned,
SortingChangedInput: built.SortingChanged,
SegmentsWithCategories: built.SegmentsWithCategories,
TimingFieldsRepaired: parsed.Stats.TimingFieldsRepaired,
TimingOrderSwapped: parsed.Stats.TimingOrderSwapped,
SpeakerFilled: parsed.Stats.SpeakerFilled,
SegmentsDroppedText: parsed.Stats.SegmentsDroppedText,
}
auditJSON, err := json.Marshal(audit)
if err != nil {
@@ -68,20 +78,24 @@ func Run(ctx context.Context, cfg config.NormalizeConfig) error {
report.Info("normalize", "normalize", "started normalize command"),
report.Info("normalize", "normalize", fmt.Sprintf("input file: %s", cfg.InputFile)),
report.Info("normalize", "normalize", fmt.Sprintf("detected input shape: %s", parsed.Shape)),
report.Info("normalize", "normalize", fmt.Sprintf("input segment count: %d", len(parsed.Segments))),
report.Info("normalize", "normalize", fmt.Sprintf("input segment count: %d", parsed.InputSegmentCount)),
report.Info("normalize", "normalize", fmt.Sprintf("selected output schema: %s", cfg.OutputSchema)),
report.Info("normalize", "normalize", fmt.Sprintf("selected output modules: %s", strings.Join(cfg.OutputModules, ","))),
report.Info("normalize", "normalize", fmt.Sprintf("output file: %s", cfg.OutputFile)),
report.Info("normalize", "normalize", fmt.Sprintf("ids reassigned: %t", built.IDsReassigned)),
report.Info("normalize", "normalize", fmt.Sprintf("sorting changed input order: %t", built.SortingChanged)),
report.Info("normalize", "normalize", fmt.Sprintf("segments with categories: %d", built.SegmentsWithCategories)),
report.Info("normalize", "normalize", fmt.Sprintf("timing fields repaired: %d", parsed.Stats.TimingFieldsRepaired)),
report.Info("normalize", "normalize", fmt.Sprintf("timing order swapped: %d", parsed.Stats.TimingOrderSwapped)),
report.Info("normalize", "normalize", fmt.Sprintf("speaker placeholders added: %d", parsed.Stats.SpeakerFilled)),
report.Info("normalize", "normalize", fmt.Sprintf("segments dropped for empty text: %d", parsed.Stats.SegmentsDroppedText)),
report.Info("normalize", "normalize-audit", string(auditJSON)),
}
if len(parsed.Segments) == 0 {
if parsed.InputSegmentCount == 0 {
events = append(events, report.Warning("normalize", "normalize", "input transcript contains zero segments"))
}
events = append(events,
report.Info("normalize", "validate-output", fmt.Sprintf("validated %d output segment(s)", len(parsed.Segments))),
report.Info("normalize", "validate-output", fmt.Sprintf("validated %d output segment(s)", built.OutputSegmentCount)),
report.Info("output", "json", "wrote transcript JSON"),
)