Updated the normalize command to correct common errors in WhisperX-generated input transcripts
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
All checks were successful
ci/woodpecker/tag/release Pipeline was successful
This commit is contained in:
16
README.md
16
README.md
@@ -142,16 +142,16 @@ Global flags:
|
||||
- Top-level object with a `segments` array.
|
||||
- Bare top-level array of segment objects (for example, Audita-style output).
|
||||
|
||||
`normalize` required segment fields:
|
||||
|
||||
- `start`
|
||||
- `end`
|
||||
- `speaker`
|
||||
- `text`
|
||||
|
||||
`normalize` behavior:
|
||||
|
||||
- Validates `start >= 0`, `end >= start`, and non-empty `speaker`.
|
||||
- Repairs missing timing fields deterministically:
|
||||
if one of `start`/`end` is present, sets both to that value;
|
||||
if both are missing, uses midpoint of previous `end` and next `start`,
|
||||
with edge fallback to available neighbor and `0.0` for single-segment inputs.
|
||||
- If `end < start`, swaps them.
|
||||
- Fills missing/empty `speaker` with `Unknown_Speaker`.
|
||||
- Drops segments with missing, empty, or whitespace-only `text`.
|
||||
- Validates repaired timing with `start >= 0`.
|
||||
- Accepts existing input `id` values as provenance only.
|
||||
- Reassigns output segment IDs sequentially from `1` to `N`.
|
||||
- Sorts deterministically by `(start, end, original_input_index, speaker)`.
|
||||
|
||||
@@ -223,7 +223,7 @@ Design constraints:
|
||||
|
||||
- `normalize` runs outside the merge pipeline and does not invoke merge preprocessing or postprocessing modules.
|
||||
- `normalize` accepts two input shapes: object-with-`segments` and bare segment arrays.
|
||||
- `normalize` validates required segment fields (`start`, `end`, `speaker`, `text`) and timing/speaker constraints.
|
||||
- `normalize` applies deterministic repair rules for missing/irregular `start`, `end`, and `speaker`, and drops segments with missing/empty `text`.
|
||||
- `normalize` sorts segments deterministically by chronological keys and stable input-index tie-breakers.
|
||||
- `normalize` assigns fresh sequential output IDs (`1..N`) after sorting.
|
||||
- `normalize` validates final output against the selected schema before writing.
|
||||
|
||||
@@ -273,6 +273,66 @@ func TestNormalizeEmptySegmentsArrayProducesValidOutput(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeRepairsAndDropsDefectiveSegments(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeJSONFile(t, dir, "input.json", `[
|
||||
{"start": 5, "speaker": "", "text": "keep-a"},
|
||||
{"end": 3, "speaker": " ", "text": "keep-b"},
|
||||
{"speaker": "A"},
|
||||
{"speaker": "A", "text": " "},
|
||||
{"start": 9, "end": 4, "speaker": "B", "text": "keep-c"}
|
||||
]`)
|
||||
output := filepath.Join(dir, "normalized.json")
|
||||
reportPath := filepath.Join(dir, "report.json")
|
||||
|
||||
err := executeNormalize(
|
||||
"--input-file", input,
|
||||
"--output-file", output,
|
||||
"--report-file", reportPath,
|
||||
"--output-schema", config.OutputSchemaIntermediate,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatalf("normalize failed: %v", err)
|
||||
}
|
||||
|
||||
var transcript schema.IntermediateTranscript
|
||||
readJSON(t, output, &transcript)
|
||||
if len(transcript.Segments) != 3 {
|
||||
t.Fatalf("segment count = %d, want 3", len(transcript.Segments))
|
||||
}
|
||||
if transcript.Segments[0].Start != 3 || transcript.Segments[0].End != 3 {
|
||||
t.Fatalf("segment[0] timing = %v..%v, want 3..3", transcript.Segments[0].Start, transcript.Segments[0].End)
|
||||
}
|
||||
if transcript.Segments[1].Start != 4 || transcript.Segments[1].End != 9 {
|
||||
t.Fatalf("segment[1] timing = %v..%v, want 4..9", transcript.Segments[1].Start, transcript.Segments[1].End)
|
||||
}
|
||||
if transcript.Segments[2].Start != 5 || transcript.Segments[2].End != 5 {
|
||||
t.Fatalf("segment[2] timing = %v..%v, want 5..5", transcript.Segments[2].Start, transcript.Segments[2].End)
|
||||
}
|
||||
if transcript.Segments[0].Speaker != "Unknown_Speaker" || transcript.Segments[2].Speaker != "Unknown_Speaker" {
|
||||
t.Fatalf("expected Unknown_Speaker placeholders, got %#v", transcript.Segments)
|
||||
}
|
||||
|
||||
var rpt report.Report
|
||||
readJSON(t, reportPath, &rpt)
|
||||
audit := extractNormalizeAudit(t, rpt)
|
||||
if audit.InputSegmentCount != 5 || audit.OutputSegmentCount != 3 {
|
||||
t.Fatalf("audit counts = in:%d out:%d, want in:5 out:3", audit.InputSegmentCount, audit.OutputSegmentCount)
|
||||
}
|
||||
if audit.TimingFieldsRepaired != 2 {
|
||||
t.Fatalf("timing fields repaired = %d, want 2", audit.TimingFieldsRepaired)
|
||||
}
|
||||
if audit.TimingOrderSwapped != 1 {
|
||||
t.Fatalf("timing order swapped = %d, want 1", audit.TimingOrderSwapped)
|
||||
}
|
||||
if audit.SpeakerFilled != 2 {
|
||||
t.Fatalf("speaker filled = %d, want 2", audit.SpeakerFilled)
|
||||
}
|
||||
if audit.SegmentsDroppedText != 2 {
|
||||
t.Fatalf("segments dropped text = %d, want 2", audit.SegmentsDroppedText)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeSelectedOutputSchemaIsHonored(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
input := writeJSONFile(t, dir, "input.json", `{"segments":[{"start":1,"end":2,"speaker":"A","text":"one"}]}`)
|
||||
@@ -434,11 +494,16 @@ type normalizeAudit struct {
|
||||
OutputFile string `json:"output_file"`
|
||||
InputShape string `json:"input_shape"`
|
||||
InputSegmentCount int `json:"input_segment_count"`
|
||||
OutputSegmentCount int `json:"output_segment_count"`
|
||||
OutputSchema string `json:"output_schema"`
|
||||
OutputModules []string `json:"output_modules"`
|
||||
IDsReassigned bool `json:"ids_reassigned"`
|
||||
SortingChangedInput bool `json:"sorting_changed_input_order"`
|
||||
SegmentsWithCategories int `json:"segments_with_categories"`
|
||||
TimingFieldsRepaired int `json:"timing_fields_repaired"`
|
||||
TimingOrderSwapped int `json:"timing_order_swapped"`
|
||||
SpeakerFilled int `json:"speaker_filled"`
|
||||
SegmentsDroppedText int `json:"segments_dropped_text"`
|
||||
}
|
||||
|
||||
func extractNormalizeAudit(t *testing.T, rpt report.Report) normalizeAudit {
|
||||
|
||||
@@ -15,6 +15,7 @@ import (
|
||||
// BuildResult contains normalize output plus deterministic transformation diagnostics.
|
||||
type BuildResult struct {
|
||||
Output any
|
||||
OutputSegmentCount int
|
||||
SortingChanged bool
|
||||
IDsReassigned bool
|
||||
SegmentsWithCategories int
|
||||
@@ -35,6 +36,7 @@ func Build(parsed ParsedTranscript, cfg config.NormalizeConfig) (BuildResult, er
|
||||
}
|
||||
return BuildResult{
|
||||
Output: output,
|
||||
OutputSegmentCount: len(ordered),
|
||||
SortingChanged: sortingChanged,
|
||||
IDsReassigned: idsReassigned,
|
||||
SegmentsWithCategories: segmentsWithCategories,
|
||||
@@ -46,6 +48,7 @@ func Build(parsed ParsedTranscript, cfg config.NormalizeConfig) (BuildResult, er
|
||||
}
|
||||
return BuildResult{
|
||||
Output: output,
|
||||
OutputSegmentCount: len(ordered),
|
||||
SortingChanged: sortingChanged,
|
||||
IDsReassigned: idsReassigned,
|
||||
SegmentsWithCategories: segmentsWithCategories,
|
||||
@@ -57,6 +60,7 @@ func Build(parsed ParsedTranscript, cfg config.NormalizeConfig) (BuildResult, er
|
||||
}
|
||||
return BuildResult{
|
||||
Output: output,
|
||||
OutputSegmentCount: len(ordered),
|
||||
SortingChanged: sortingChanged,
|
||||
IDsReassigned: idsReassigned,
|
||||
SegmentsWithCategories: segmentsWithCategories,
|
||||
|
||||
@@ -19,11 +19,16 @@ type normalizeAudit struct {
|
||||
OutputFile string `json:"output_file"`
|
||||
InputShape string `json:"input_shape"`
|
||||
InputSegmentCount int `json:"input_segment_count"`
|
||||
OutputSegmentCount int `json:"output_segment_count"`
|
||||
OutputSchema string `json:"output_schema"`
|
||||
OutputModules []string `json:"output_modules"`
|
||||
IDsReassigned bool `json:"ids_reassigned"`
|
||||
SortingChangedInput bool `json:"sorting_changed_input_order"`
|
||||
SegmentsWithCategories int `json:"segments_with_categories"`
|
||||
TimingFieldsRepaired int `json:"timing_fields_repaired"`
|
||||
TimingOrderSwapped int `json:"timing_order_swapped"`
|
||||
SpeakerFilled int `json:"speaker_filled"`
|
||||
SegmentsDroppedText int `json:"segments_dropped_text"`
|
||||
}
|
||||
|
||||
// Run executes artifact-level normalization.
|
||||
@@ -52,12 +57,17 @@ func Run(ctx context.Context, cfg config.NormalizeConfig) error {
|
||||
InputFile: cfg.InputFile,
|
||||
OutputFile: cfg.OutputFile,
|
||||
InputShape: string(parsed.Shape),
|
||||
InputSegmentCount: len(parsed.Segments),
|
||||
InputSegmentCount: parsed.InputSegmentCount,
|
||||
OutputSegmentCount: built.OutputSegmentCount,
|
||||
OutputSchema: cfg.OutputSchema,
|
||||
OutputModules: append([]string(nil), cfg.OutputModules...),
|
||||
IDsReassigned: built.IDsReassigned,
|
||||
SortingChangedInput: built.SortingChanged,
|
||||
SegmentsWithCategories: built.SegmentsWithCategories,
|
||||
TimingFieldsRepaired: parsed.Stats.TimingFieldsRepaired,
|
||||
TimingOrderSwapped: parsed.Stats.TimingOrderSwapped,
|
||||
SpeakerFilled: parsed.Stats.SpeakerFilled,
|
||||
SegmentsDroppedText: parsed.Stats.SegmentsDroppedText,
|
||||
}
|
||||
auditJSON, err := json.Marshal(audit)
|
||||
if err != nil {
|
||||
@@ -68,20 +78,24 @@ func Run(ctx context.Context, cfg config.NormalizeConfig) error {
|
||||
report.Info("normalize", "normalize", "started normalize command"),
|
||||
report.Info("normalize", "normalize", fmt.Sprintf("input file: %s", cfg.InputFile)),
|
||||
report.Info("normalize", "normalize", fmt.Sprintf("detected input shape: %s", parsed.Shape)),
|
||||
report.Info("normalize", "normalize", fmt.Sprintf("input segment count: %d", len(parsed.Segments))),
|
||||
report.Info("normalize", "normalize", fmt.Sprintf("input segment count: %d", parsed.InputSegmentCount)),
|
||||
report.Info("normalize", "normalize", fmt.Sprintf("selected output schema: %s", cfg.OutputSchema)),
|
||||
report.Info("normalize", "normalize", fmt.Sprintf("selected output modules: %s", strings.Join(cfg.OutputModules, ","))),
|
||||
report.Info("normalize", "normalize", fmt.Sprintf("output file: %s", cfg.OutputFile)),
|
||||
report.Info("normalize", "normalize", fmt.Sprintf("ids reassigned: %t", built.IDsReassigned)),
|
||||
report.Info("normalize", "normalize", fmt.Sprintf("sorting changed input order: %t", built.SortingChanged)),
|
||||
report.Info("normalize", "normalize", fmt.Sprintf("segments with categories: %d", built.SegmentsWithCategories)),
|
||||
report.Info("normalize", "normalize", fmt.Sprintf("timing fields repaired: %d", parsed.Stats.TimingFieldsRepaired)),
|
||||
report.Info("normalize", "normalize", fmt.Sprintf("timing order swapped: %d", parsed.Stats.TimingOrderSwapped)),
|
||||
report.Info("normalize", "normalize", fmt.Sprintf("speaker placeholders added: %d", parsed.Stats.SpeakerFilled)),
|
||||
report.Info("normalize", "normalize", fmt.Sprintf("segments dropped for empty text: %d", parsed.Stats.SegmentsDroppedText)),
|
||||
report.Info("normalize", "normalize-audit", string(auditJSON)),
|
||||
}
|
||||
if len(parsed.Segments) == 0 {
|
||||
if parsed.InputSegmentCount == 0 {
|
||||
events = append(events, report.Warning("normalize", "normalize", "input transcript contains zero segments"))
|
||||
}
|
||||
events = append(events,
|
||||
report.Info("normalize", "validate-output", fmt.Sprintf("validated %d output segment(s)", len(parsed.Segments))),
|
||||
report.Info("normalize", "validate-output", fmt.Sprintf("validated %d output segment(s)", built.OutputSegmentCount)),
|
||||
report.Info("output", "json", "wrote transcript JSON"),
|
||||
)
|
||||
|
||||
|
||||
@@ -19,14 +19,20 @@ const (
|
||||
|
||||
// ParsedTranscript is the validated normalize input model.
|
||||
type ParsedTranscript struct {
|
||||
Shape InputShape
|
||||
Segments []InputSegment
|
||||
Shape InputShape
|
||||
InputSegmentCount int
|
||||
Stats NormalizeStats
|
||||
Segments []InputSegment
|
||||
}
|
||||
|
||||
// InputSegment is a validated segment from normalize input.
|
||||
type InputSegment struct {
|
||||
InputIndex int
|
||||
OriginalID *int
|
||||
StartPresent bool
|
||||
EndPresent bool
|
||||
SpeakerPresent bool
|
||||
TextPresent bool
|
||||
Start float64
|
||||
End float64
|
||||
Speaker string
|
||||
@@ -39,6 +45,14 @@ type InputSegment struct {
|
||||
OverlapGroupID *int
|
||||
}
|
||||
|
||||
// NormalizeStats captures deterministic repair/drop outcomes from normalize input processing.
|
||||
type NormalizeStats struct {
|
||||
TimingFieldsRepaired int
|
||||
TimingOrderSwapped int
|
||||
SpeakerFilled int
|
||||
SegmentsDroppedText int
|
||||
}
|
||||
|
||||
type inputSegmentPayload struct {
|
||||
ID *int `json:"id"`
|
||||
Start *float64 `json:"start"`
|
||||
@@ -85,13 +99,15 @@ func ParseReader(reader io.Reader) (ParsedTranscript, error) {
|
||||
case '{':
|
||||
return parseObjectShape(trimmed)
|
||||
case '[':
|
||||
segments, err := parseSegmentsArray(trimmed)
|
||||
segments, stats, inputCount, err := parseSegmentsArray(trimmed)
|
||||
if err != nil {
|
||||
return ParsedTranscript{}, err
|
||||
}
|
||||
return ParsedTranscript{
|
||||
Shape: ShapeBareSegmentsArray,
|
||||
Segments: segments,
|
||||
Shape: ShapeBareSegmentsArray,
|
||||
InputSegmentCount: inputCount,
|
||||
Stats: stats,
|
||||
Segments: segments,
|
||||
}, nil
|
||||
default:
|
||||
return ParsedTranscript{}, fmt.Errorf("normalize input must be a top-level object with \"segments\" or a top-level segment array")
|
||||
@@ -121,72 +137,73 @@ func parseObjectShape(raw []byte) (ParsedTranscript, error) {
|
||||
return ParsedTranscript{}, fmt.Errorf("normalize object input must contain a \"segments\" field")
|
||||
}
|
||||
|
||||
segments, err := parseSegmentsArray(segmentsRaw)
|
||||
segments, stats, inputCount, err := parseSegmentsArray(segmentsRaw)
|
||||
if err != nil {
|
||||
return ParsedTranscript{}, err
|
||||
}
|
||||
|
||||
return ParsedTranscript{
|
||||
Shape: ShapeObjectWithSegments,
|
||||
Segments: segments,
|
||||
Shape: ShapeObjectWithSegments,
|
||||
InputSegmentCount: inputCount,
|
||||
Stats: stats,
|
||||
Segments: segments,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func parseSegmentsArray(raw []byte) ([]InputSegment, error) {
|
||||
func parseSegmentsArray(raw []byte) ([]InputSegment, NormalizeStats, int, error) {
|
||||
var segmentValues []json.RawMessage
|
||||
if err := json.Unmarshal(raw, &segmentValues); err != nil {
|
||||
return nil, fmt.Errorf("normalize input \"segments\" must be an array")
|
||||
return nil, NormalizeStats{}, 0, fmt.Errorf("normalize input \"segments\" must be an array")
|
||||
}
|
||||
|
||||
segments := make([]InputSegment, len(segmentValues))
|
||||
for index, segmentRaw := range segmentValues {
|
||||
segment, err := parseSegment(index, segmentRaw)
|
||||
segment, err := decodeSegment(index, segmentRaw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
return nil, NormalizeStats{}, 0, err
|
||||
}
|
||||
segments[index] = segment
|
||||
}
|
||||
return segments, nil
|
||||
normalized, stats, err := normalizeSegments(segments)
|
||||
return normalized, stats, len(segmentValues), err
|
||||
}
|
||||
|
||||
func parseSegment(index int, raw []byte) (InputSegment, error) {
|
||||
func decodeSegment(index int, raw []byte) (InputSegment, error) {
|
||||
var payload inputSegmentPayload
|
||||
if err := json.Unmarshal(raw, &payload); err != nil {
|
||||
return InputSegment{}, fmt.Errorf("segment %d: invalid segment object: %w", index, err)
|
||||
}
|
||||
|
||||
if payload.Start == nil {
|
||||
return InputSegment{}, fmt.Errorf("segment %d is missing required field \"start\"", index)
|
||||
var start float64
|
||||
if payload.Start != nil {
|
||||
start = *payload.Start
|
||||
}
|
||||
if payload.End == nil {
|
||||
return InputSegment{}, fmt.Errorf("segment %d is missing required field \"end\"", index)
|
||||
}
|
||||
if payload.Speaker == nil {
|
||||
return InputSegment{}, fmt.Errorf("segment %d is missing required field \"speaker\"", index)
|
||||
}
|
||||
if payload.Text == nil {
|
||||
return InputSegment{}, fmt.Errorf("segment %d is missing required field \"text\"", index)
|
||||
var end float64
|
||||
if payload.End != nil {
|
||||
end = *payload.End
|
||||
}
|
||||
|
||||
if *payload.Start < 0 {
|
||||
return InputSegment{}, fmt.Errorf("segment %d has start %v; start must be >= 0", index, *payload.Start)
|
||||
}
|
||||
if *payload.End < *payload.Start {
|
||||
return InputSegment{}, fmt.Errorf("segment %d has end %v before start %v", index, *payload.End, *payload.Start)
|
||||
speaker := ""
|
||||
if payload.Speaker != nil {
|
||||
speaker = strings.TrimSpace(*payload.Speaker)
|
||||
}
|
||||
|
||||
speaker := strings.TrimSpace(*payload.Speaker)
|
||||
if speaker == "" {
|
||||
return InputSegment{}, fmt.Errorf("segment %d has empty \"speaker\"; speaker must be non-empty", index)
|
||||
text := ""
|
||||
if payload.Text != nil {
|
||||
text = *payload.Text
|
||||
}
|
||||
|
||||
return InputSegment{
|
||||
InputIndex: index,
|
||||
OriginalID: payload.ID,
|
||||
Start: *payload.Start,
|
||||
End: *payload.End,
|
||||
StartPresent: payload.Start != nil,
|
||||
EndPresent: payload.End != nil,
|
||||
SpeakerPresent: payload.Speaker != nil,
|
||||
TextPresent: payload.Text != nil,
|
||||
Start: start,
|
||||
End: end,
|
||||
Speaker: speaker,
|
||||
Text: *payload.Text,
|
||||
Text: text,
|
||||
Categories: append([]string(nil), payload.Categories...),
|
||||
Source: payload.Source,
|
||||
SourceSegmentIndex: payload.SourceSegmentIndex,
|
||||
@@ -195,3 +212,96 @@ func parseSegment(index int, raw []byte) (InputSegment, error) {
|
||||
OverlapGroupID: payload.OverlapGroupID,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func normalizeSegments(segments []InputSegment) ([]InputSegment, NormalizeStats, error) {
|
||||
stats := NormalizeStats{}
|
||||
filtered := make([]InputSegment, 0, len(segments))
|
||||
for _, segment := range segments {
|
||||
if strings.TrimSpace(segment.Text) == "" {
|
||||
stats.SegmentsDroppedText++
|
||||
continue
|
||||
}
|
||||
filtered = append(filtered, segment)
|
||||
}
|
||||
|
||||
if len(filtered) == 0 {
|
||||
return filtered, stats, nil
|
||||
}
|
||||
|
||||
hasStart := make([]bool, len(filtered))
|
||||
hasEnd := make([]bool, len(filtered))
|
||||
for index, segment := range filtered {
|
||||
hasStart[index] = segment.StartPresent
|
||||
hasEnd[index] = segment.EndPresent
|
||||
}
|
||||
for index := range filtered {
|
||||
if !hasStart[index] && !hasEnd[index] {
|
||||
midpoint := inferTimestampFromNeighbors(filtered, hasStart, hasEnd, index)
|
||||
filtered[index].Start = midpoint
|
||||
filtered[index].End = midpoint
|
||||
stats.TimingFieldsRepaired += 2
|
||||
hasStart[index] = true
|
||||
hasEnd[index] = true
|
||||
continue
|
||||
}
|
||||
if hasStart[index] && !hasEnd[index] {
|
||||
filtered[index].End = filtered[index].Start
|
||||
stats.TimingFieldsRepaired++
|
||||
hasEnd[index] = true
|
||||
}
|
||||
if !hasStart[index] && hasEnd[index] {
|
||||
filtered[index].Start = filtered[index].End
|
||||
stats.TimingFieldsRepaired++
|
||||
hasStart[index] = true
|
||||
}
|
||||
}
|
||||
|
||||
for index := range filtered {
|
||||
if filtered[index].Start < 0 {
|
||||
return nil, NormalizeStats{}, fmt.Errorf("segment %d has start %v; start must be >= 0", filtered[index].InputIndex, filtered[index].Start)
|
||||
}
|
||||
if filtered[index].End < filtered[index].Start {
|
||||
filtered[index].Start, filtered[index].End = filtered[index].End, filtered[index].Start
|
||||
stats.TimingOrderSwapped++
|
||||
}
|
||||
if strings.TrimSpace(filtered[index].Speaker) == "" {
|
||||
filtered[index].Speaker = "Unknown_Speaker"
|
||||
stats.SpeakerFilled++
|
||||
}
|
||||
}
|
||||
|
||||
return filtered, stats, nil
|
||||
}
|
||||
|
||||
func inferTimestampFromNeighbors(segments []InputSegment, hasStart []bool, hasEnd []bool, index int) float64 {
|
||||
prevEnd, hasPrev := nearestPreviousEnd(segments, hasEnd, index)
|
||||
nextStart, hasNext := nearestNextStart(segments, hasStart, index)
|
||||
if hasPrev && hasNext {
|
||||
return (prevEnd + nextStart) / 2
|
||||
}
|
||||
if hasPrev {
|
||||
return prevEnd
|
||||
}
|
||||
if hasNext {
|
||||
return nextStart
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func nearestPreviousEnd(segments []InputSegment, hasEnd []bool, index int) (float64, bool) {
|
||||
for i := index - 1; i >= 0; i-- {
|
||||
if hasEnd[i] {
|
||||
return segments[i].End, true
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func nearestNextStart(segments []InputSegment, hasStart []bool, index int) (float64, bool) {
|
||||
for i := index + 1; i < len(segments); i++ {
|
||||
if hasStart[i] {
|
||||
return segments[i].Start, true
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
@@ -92,34 +92,70 @@ func TestParseReaderTopLevelScalarShapesFail(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReaderMissingStartFails(t *testing.T) {
|
||||
_, err := ParseReader(strings.NewReader(`[{"end":2,"speaker":"A","text":"t"}]`))
|
||||
assertContains(t, err, `missing required field "start"`)
|
||||
func TestParseReaderMissingStartUsesEndValue(t *testing.T) {
|
||||
parsed, err := ParseReader(strings.NewReader(`[{"end":2,"speaker":"A","text":"t"}]`))
|
||||
if err != nil {
|
||||
t.Fatalf("parse failed: %v", err)
|
||||
}
|
||||
if len(parsed.Segments) != 1 {
|
||||
t.Fatalf("segment count = %d, want 1", len(parsed.Segments))
|
||||
}
|
||||
if parsed.Segments[0].Start != 2 || parsed.Segments[0].End != 2 {
|
||||
t.Fatalf("segment timing = %v..%v, want 2..2", parsed.Segments[0].Start, parsed.Segments[0].End)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReaderMissingEndFails(t *testing.T) {
|
||||
_, err := ParseReader(strings.NewReader(`[{"start":1,"speaker":"A","text":"t"}]`))
|
||||
assertContains(t, err, `missing required field "end"`)
|
||||
func TestParseReaderMissingEndUsesStartValue(t *testing.T) {
|
||||
parsed, err := ParseReader(strings.NewReader(`[{"start":1,"speaker":"A","text":"t"}]`))
|
||||
if err != nil {
|
||||
t.Fatalf("parse failed: %v", err)
|
||||
}
|
||||
if len(parsed.Segments) != 1 {
|
||||
t.Fatalf("segment count = %d, want 1", len(parsed.Segments))
|
||||
}
|
||||
if parsed.Segments[0].Start != 1 || parsed.Segments[0].End != 1 {
|
||||
t.Fatalf("segment timing = %v..%v, want 1..1", parsed.Segments[0].Start, parsed.Segments[0].End)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReaderMissingSpeakerFails(t *testing.T) {
|
||||
_, err := ParseReader(strings.NewReader(`[{"start":1,"end":2,"text":"t"}]`))
|
||||
assertContains(t, err, `missing required field "speaker"`)
|
||||
func TestParseReaderMissingSpeakerUsesUnknownPlaceholder(t *testing.T) {
|
||||
parsed, err := ParseReader(strings.NewReader(`[{"start":1,"end":2,"text":"t"}]`))
|
||||
if err != nil {
|
||||
t.Fatalf("parse failed: %v", err)
|
||||
}
|
||||
if parsed.Segments[0].Speaker != "Unknown_Speaker" {
|
||||
t.Fatalf("speaker = %q, want Unknown_Speaker", parsed.Segments[0].Speaker)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReaderEmptySpeakerFails(t *testing.T) {
|
||||
_, err := ParseReader(strings.NewReader(`[{"start":1,"end":2,"speaker":" ","text":"t"}]`))
|
||||
assertContains(t, err, `speaker must be non-empty`)
|
||||
func TestParseReaderEmptySpeakerUsesUnknownPlaceholder(t *testing.T) {
|
||||
parsed, err := ParseReader(strings.NewReader(`[{"start":1,"end":2,"speaker":" ","text":"t"}]`))
|
||||
if err != nil {
|
||||
t.Fatalf("parse failed: %v", err)
|
||||
}
|
||||
if parsed.Segments[0].Speaker != "Unknown_Speaker" {
|
||||
t.Fatalf("speaker = %q, want Unknown_Speaker", parsed.Segments[0].Speaker)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReaderMissingTextFails(t *testing.T) {
|
||||
_, err := ParseReader(strings.NewReader(`[{"start":1,"end":2,"speaker":"A"}]`))
|
||||
assertContains(t, err, `missing required field "text"`)
|
||||
func TestParseReaderMissingTextDropsSegment(t *testing.T) {
|
||||
parsed, err := ParseReader(strings.NewReader(`[{"start":1,"end":2,"speaker":"A"}]`))
|
||||
if err != nil {
|
||||
t.Fatalf("parse failed: %v", err)
|
||||
}
|
||||
if len(parsed.Segments) != 0 {
|
||||
t.Fatalf("segment count = %d, want 0", len(parsed.Segments))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReaderEndBeforeStartFails(t *testing.T) {
|
||||
_, err := ParseReader(strings.NewReader(`[{"start":3,"end":2,"speaker":"A","text":"t"}]`))
|
||||
assertContains(t, err, "before start")
|
||||
func TestParseReaderEndBeforeStartSwapsValues(t *testing.T) {
|
||||
parsed, err := ParseReader(strings.NewReader(`[{"start":3,"end":2,"speaker":"A","text":"t"}]`))
|
||||
if err != nil {
|
||||
t.Fatalf("parse failed: %v", err)
|
||||
}
|
||||
if parsed.Segments[0].Start != 2 || parsed.Segments[0].End != 3 {
|
||||
t.Fatalf("segment timing = %v..%v, want 2..3", parsed.Segments[0].Start, parsed.Segments[0].End)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReaderNegativeStartFails(t *testing.T) {
|
||||
@@ -153,6 +189,68 @@ func TestParseReaderCategoriesPreservedWhenValid(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReaderMissingBothTimesMiddleUsesNeighborMidpoint(t *testing.T) {
|
||||
parsed, err := ParseReader(strings.NewReader(`[
|
||||
{"start":1,"end":2,"speaker":"A","text":"left"},
|
||||
{"speaker":"B","text":"middle"},
|
||||
{"start":6,"end":7,"speaker":"C","text":"right"}
|
||||
]`))
|
||||
if err != nil {
|
||||
t.Fatalf("parse failed: %v", err)
|
||||
}
|
||||
if parsed.Segments[1].Start != 4 || parsed.Segments[1].End != 4 {
|
||||
t.Fatalf("middle timing = %v..%v, want 4..4", parsed.Segments[1].Start, parsed.Segments[1].End)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReaderMissingBothTimesEdgeFallbacks(t *testing.T) {
|
||||
parsedFirst, err := ParseReader(strings.NewReader(`[
|
||||
{"speaker":"A","text":"first"},
|
||||
{"start":5,"end":6,"speaker":"B","text":"second"}
|
||||
]`))
|
||||
if err != nil {
|
||||
t.Fatalf("parse first failed: %v", err)
|
||||
}
|
||||
if parsedFirst.Segments[0].Start != 5 || parsedFirst.Segments[0].End != 5 {
|
||||
t.Fatalf("first timing = %v..%v, want 5..5", parsedFirst.Segments[0].Start, parsedFirst.Segments[0].End)
|
||||
}
|
||||
|
||||
parsedLast, err := ParseReader(strings.NewReader(`[
|
||||
{"start":1,"end":2,"speaker":"A","text":"first"},
|
||||
{"speaker":"B","text":"last"}
|
||||
]`))
|
||||
if err != nil {
|
||||
t.Fatalf("parse last failed: %v", err)
|
||||
}
|
||||
if parsedLast.Segments[1].Start != 2 || parsedLast.Segments[1].End != 2 {
|
||||
t.Fatalf("last timing = %v..%v, want 2..2", parsedLast.Segments[1].Start, parsedLast.Segments[1].End)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReaderMissingBothTimesSingleSegmentUsesZero(t *testing.T) {
|
||||
parsed, err := ParseReader(strings.NewReader(`[{"speaker":"A","text":"only"}]`))
|
||||
if err != nil {
|
||||
t.Fatalf("parse failed: %v", err)
|
||||
}
|
||||
if parsed.Segments[0].Start != 0 || parsed.Segments[0].End != 0 {
|
||||
t.Fatalf("timing = %v..%v, want 0..0", parsed.Segments[0].Start, parsed.Segments[0].End)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReaderEmptyOrWhitespaceTextDropped(t *testing.T) {
|
||||
parsed, err := ParseReader(strings.NewReader(`[
|
||||
{"start":1,"end":2,"speaker":"A","text":"ok"},
|
||||
{"start":2,"end":3,"speaker":"A","text":""},
|
||||
{"start":3,"end":4,"speaker":"A","text":" "}
|
||||
]`))
|
||||
if err != nil {
|
||||
t.Fatalf("parse failed: %v", err)
|
||||
}
|
||||
if len(parsed.Segments) != 1 {
|
||||
t.Fatalf("segment count = %d, want 1", len(parsed.Segments))
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseReaderOriginalInputIndexPreserved(t *testing.T) {
|
||||
input := `[
|
||||
{"start":1,"end":2,"speaker":"A","text":"one"},
|
||||
|
||||
Reference in New Issue
Block a user