Compare commits
1 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| b20438acf0 |
16
README.md
16
README.md
@@ -142,16 +142,16 @@ Global flags:
|
|||||||
- Top-level object with a `segments` array.
|
- Top-level object with a `segments` array.
|
||||||
- Bare top-level array of segment objects (for example, Audita-style output).
|
- Bare top-level array of segment objects (for example, Audita-style output).
|
||||||
|
|
||||||
`normalize` required segment fields:
|
|
||||||
|
|
||||||
- `start`
|
|
||||||
- `end`
|
|
||||||
- `speaker`
|
|
||||||
- `text`
|
|
||||||
|
|
||||||
`normalize` behavior:
|
`normalize` behavior:
|
||||||
|
|
||||||
- Validates `start >= 0`, `end >= start`, and non-empty `speaker`.
|
- Repairs missing timing fields deterministically:
|
||||||
|
if one of `start`/`end` is present, sets both to that value;
|
||||||
|
if both are missing, uses midpoint of previous `end` and next `start`,
|
||||||
|
with edge fallback to available neighbor and `0.0` for single-segment inputs.
|
||||||
|
- If `end < start`, swaps them.
|
||||||
|
- Fills missing/empty `speaker` with `Unknown_Speaker`.
|
||||||
|
- Drops segments with missing, empty, or whitespace-only `text`.
|
||||||
|
- Validates repaired timing with `start >= 0`.
|
||||||
- Accepts existing input `id` values as provenance only.
|
- Accepts existing input `id` values as provenance only.
|
||||||
- Reassigns output segment IDs sequentially from `1` to `N`.
|
- Reassigns output segment IDs sequentially from `1` to `N`.
|
||||||
- Sorts deterministically by `(start, end, original_input_index, speaker)`.
|
- Sorts deterministically by `(start, end, original_input_index, speaker)`.
|
||||||
|
|||||||
@@ -223,7 +223,7 @@ Design constraints:
|
|||||||
|
|
||||||
- `normalize` runs outside the merge pipeline and does not invoke merge preprocessing or postprocessing modules.
|
- `normalize` runs outside the merge pipeline and does not invoke merge preprocessing or postprocessing modules.
|
||||||
- `normalize` accepts two input shapes: object-with-`segments` and bare segment arrays.
|
- `normalize` accepts two input shapes: object-with-`segments` and bare segment arrays.
|
||||||
- `normalize` validates required segment fields (`start`, `end`, `speaker`, `text`) and timing/speaker constraints.
|
- `normalize` applies deterministic repair rules for missing/irregular `start`, `end`, and `speaker`, and drops segments with missing/empty `text`.
|
||||||
- `normalize` sorts segments deterministically by chronological keys and stable input-index tie-breakers.
|
- `normalize` sorts segments deterministically by chronological keys and stable input-index tie-breakers.
|
||||||
- `normalize` assigns fresh sequential output IDs (`1..N`) after sorting.
|
- `normalize` assigns fresh sequential output IDs (`1..N`) after sorting.
|
||||||
- `normalize` validates final output against the selected schema before writing.
|
- `normalize` validates final output against the selected schema before writing.
|
||||||
|
|||||||
@@ -273,6 +273,66 @@ func TestNormalizeEmptySegmentsArrayProducesValidOutput(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestNormalizeRepairsAndDropsDefectiveSegments(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
input := writeJSONFile(t, dir, "input.json", `[
|
||||||
|
{"start": 5, "speaker": "", "text": "keep-a"},
|
||||||
|
{"end": 3, "speaker": " ", "text": "keep-b"},
|
||||||
|
{"speaker": "A"},
|
||||||
|
{"speaker": "A", "text": " "},
|
||||||
|
{"start": 9, "end": 4, "speaker": "B", "text": "keep-c"}
|
||||||
|
]`)
|
||||||
|
output := filepath.Join(dir, "normalized.json")
|
||||||
|
reportPath := filepath.Join(dir, "report.json")
|
||||||
|
|
||||||
|
err := executeNormalize(
|
||||||
|
"--input-file", input,
|
||||||
|
"--output-file", output,
|
||||||
|
"--report-file", reportPath,
|
||||||
|
"--output-schema", config.OutputSchemaIntermediate,
|
||||||
|
)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("normalize failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var transcript schema.IntermediateTranscript
|
||||||
|
readJSON(t, output, &transcript)
|
||||||
|
if len(transcript.Segments) != 3 {
|
||||||
|
t.Fatalf("segment count = %d, want 3", len(transcript.Segments))
|
||||||
|
}
|
||||||
|
if transcript.Segments[0].Start != 3 || transcript.Segments[0].End != 3 {
|
||||||
|
t.Fatalf("segment[0] timing = %v..%v, want 3..3", transcript.Segments[0].Start, transcript.Segments[0].End)
|
||||||
|
}
|
||||||
|
if transcript.Segments[1].Start != 4 || transcript.Segments[1].End != 9 {
|
||||||
|
t.Fatalf("segment[1] timing = %v..%v, want 4..9", transcript.Segments[1].Start, transcript.Segments[1].End)
|
||||||
|
}
|
||||||
|
if transcript.Segments[2].Start != 5 || transcript.Segments[2].End != 5 {
|
||||||
|
t.Fatalf("segment[2] timing = %v..%v, want 5..5", transcript.Segments[2].Start, transcript.Segments[2].End)
|
||||||
|
}
|
||||||
|
if transcript.Segments[0].Speaker != "Unknown_Speaker" || transcript.Segments[2].Speaker != "Unknown_Speaker" {
|
||||||
|
t.Fatalf("expected Unknown_Speaker placeholders, got %#v", transcript.Segments)
|
||||||
|
}
|
||||||
|
|
||||||
|
var rpt report.Report
|
||||||
|
readJSON(t, reportPath, &rpt)
|
||||||
|
audit := extractNormalizeAudit(t, rpt)
|
||||||
|
if audit.InputSegmentCount != 5 || audit.OutputSegmentCount != 3 {
|
||||||
|
t.Fatalf("audit counts = in:%d out:%d, want in:5 out:3", audit.InputSegmentCount, audit.OutputSegmentCount)
|
||||||
|
}
|
||||||
|
if audit.TimingFieldsRepaired != 2 {
|
||||||
|
t.Fatalf("timing fields repaired = %d, want 2", audit.TimingFieldsRepaired)
|
||||||
|
}
|
||||||
|
if audit.TimingOrderSwapped != 1 {
|
||||||
|
t.Fatalf("timing order swapped = %d, want 1", audit.TimingOrderSwapped)
|
||||||
|
}
|
||||||
|
if audit.SpeakerFilled != 2 {
|
||||||
|
t.Fatalf("speaker filled = %d, want 2", audit.SpeakerFilled)
|
||||||
|
}
|
||||||
|
if audit.SegmentsDroppedText != 2 {
|
||||||
|
t.Fatalf("segments dropped text = %d, want 2", audit.SegmentsDroppedText)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestNormalizeSelectedOutputSchemaIsHonored(t *testing.T) {
|
func TestNormalizeSelectedOutputSchemaIsHonored(t *testing.T) {
|
||||||
dir := t.TempDir()
|
dir := t.TempDir()
|
||||||
input := writeJSONFile(t, dir, "input.json", `{"segments":[{"start":1,"end":2,"speaker":"A","text":"one"}]}`)
|
input := writeJSONFile(t, dir, "input.json", `{"segments":[{"start":1,"end":2,"speaker":"A","text":"one"}]}`)
|
||||||
@@ -434,11 +494,16 @@ type normalizeAudit struct {
|
|||||||
OutputFile string `json:"output_file"`
|
OutputFile string `json:"output_file"`
|
||||||
InputShape string `json:"input_shape"`
|
InputShape string `json:"input_shape"`
|
||||||
InputSegmentCount int `json:"input_segment_count"`
|
InputSegmentCount int `json:"input_segment_count"`
|
||||||
|
OutputSegmentCount int `json:"output_segment_count"`
|
||||||
OutputSchema string `json:"output_schema"`
|
OutputSchema string `json:"output_schema"`
|
||||||
OutputModules []string `json:"output_modules"`
|
OutputModules []string `json:"output_modules"`
|
||||||
IDsReassigned bool `json:"ids_reassigned"`
|
IDsReassigned bool `json:"ids_reassigned"`
|
||||||
SortingChangedInput bool `json:"sorting_changed_input_order"`
|
SortingChangedInput bool `json:"sorting_changed_input_order"`
|
||||||
SegmentsWithCategories int `json:"segments_with_categories"`
|
SegmentsWithCategories int `json:"segments_with_categories"`
|
||||||
|
TimingFieldsRepaired int `json:"timing_fields_repaired"`
|
||||||
|
TimingOrderSwapped int `json:"timing_order_swapped"`
|
||||||
|
SpeakerFilled int `json:"speaker_filled"`
|
||||||
|
SegmentsDroppedText int `json:"segments_dropped_text"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func extractNormalizeAudit(t *testing.T, rpt report.Report) normalizeAudit {
|
func extractNormalizeAudit(t *testing.T, rpt report.Report) normalizeAudit {
|
||||||
|
|||||||
@@ -15,6 +15,7 @@ import (
|
|||||||
// BuildResult contains normalize output plus deterministic transformation diagnostics.
|
// BuildResult contains normalize output plus deterministic transformation diagnostics.
|
||||||
type BuildResult struct {
|
type BuildResult struct {
|
||||||
Output any
|
Output any
|
||||||
|
OutputSegmentCount int
|
||||||
SortingChanged bool
|
SortingChanged bool
|
||||||
IDsReassigned bool
|
IDsReassigned bool
|
||||||
SegmentsWithCategories int
|
SegmentsWithCategories int
|
||||||
@@ -35,6 +36,7 @@ func Build(parsed ParsedTranscript, cfg config.NormalizeConfig) (BuildResult, er
|
|||||||
}
|
}
|
||||||
return BuildResult{
|
return BuildResult{
|
||||||
Output: output,
|
Output: output,
|
||||||
|
OutputSegmentCount: len(ordered),
|
||||||
SortingChanged: sortingChanged,
|
SortingChanged: sortingChanged,
|
||||||
IDsReassigned: idsReassigned,
|
IDsReassigned: idsReassigned,
|
||||||
SegmentsWithCategories: segmentsWithCategories,
|
SegmentsWithCategories: segmentsWithCategories,
|
||||||
@@ -46,6 +48,7 @@ func Build(parsed ParsedTranscript, cfg config.NormalizeConfig) (BuildResult, er
|
|||||||
}
|
}
|
||||||
return BuildResult{
|
return BuildResult{
|
||||||
Output: output,
|
Output: output,
|
||||||
|
OutputSegmentCount: len(ordered),
|
||||||
SortingChanged: sortingChanged,
|
SortingChanged: sortingChanged,
|
||||||
IDsReassigned: idsReassigned,
|
IDsReassigned: idsReassigned,
|
||||||
SegmentsWithCategories: segmentsWithCategories,
|
SegmentsWithCategories: segmentsWithCategories,
|
||||||
@@ -57,6 +60,7 @@ func Build(parsed ParsedTranscript, cfg config.NormalizeConfig) (BuildResult, er
|
|||||||
}
|
}
|
||||||
return BuildResult{
|
return BuildResult{
|
||||||
Output: output,
|
Output: output,
|
||||||
|
OutputSegmentCount: len(ordered),
|
||||||
SortingChanged: sortingChanged,
|
SortingChanged: sortingChanged,
|
||||||
IDsReassigned: idsReassigned,
|
IDsReassigned: idsReassigned,
|
||||||
SegmentsWithCategories: segmentsWithCategories,
|
SegmentsWithCategories: segmentsWithCategories,
|
||||||
|
|||||||
@@ -19,11 +19,16 @@ type normalizeAudit struct {
|
|||||||
OutputFile string `json:"output_file"`
|
OutputFile string `json:"output_file"`
|
||||||
InputShape string `json:"input_shape"`
|
InputShape string `json:"input_shape"`
|
||||||
InputSegmentCount int `json:"input_segment_count"`
|
InputSegmentCount int `json:"input_segment_count"`
|
||||||
|
OutputSegmentCount int `json:"output_segment_count"`
|
||||||
OutputSchema string `json:"output_schema"`
|
OutputSchema string `json:"output_schema"`
|
||||||
OutputModules []string `json:"output_modules"`
|
OutputModules []string `json:"output_modules"`
|
||||||
IDsReassigned bool `json:"ids_reassigned"`
|
IDsReassigned bool `json:"ids_reassigned"`
|
||||||
SortingChangedInput bool `json:"sorting_changed_input_order"`
|
SortingChangedInput bool `json:"sorting_changed_input_order"`
|
||||||
SegmentsWithCategories int `json:"segments_with_categories"`
|
SegmentsWithCategories int `json:"segments_with_categories"`
|
||||||
|
TimingFieldsRepaired int `json:"timing_fields_repaired"`
|
||||||
|
TimingOrderSwapped int `json:"timing_order_swapped"`
|
||||||
|
SpeakerFilled int `json:"speaker_filled"`
|
||||||
|
SegmentsDroppedText int `json:"segments_dropped_text"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// Run executes artifact-level normalization.
|
// Run executes artifact-level normalization.
|
||||||
@@ -52,12 +57,17 @@ func Run(ctx context.Context, cfg config.NormalizeConfig) error {
|
|||||||
InputFile: cfg.InputFile,
|
InputFile: cfg.InputFile,
|
||||||
OutputFile: cfg.OutputFile,
|
OutputFile: cfg.OutputFile,
|
||||||
InputShape: string(parsed.Shape),
|
InputShape: string(parsed.Shape),
|
||||||
InputSegmentCount: len(parsed.Segments),
|
InputSegmentCount: parsed.InputSegmentCount,
|
||||||
|
OutputSegmentCount: built.OutputSegmentCount,
|
||||||
OutputSchema: cfg.OutputSchema,
|
OutputSchema: cfg.OutputSchema,
|
||||||
OutputModules: append([]string(nil), cfg.OutputModules...),
|
OutputModules: append([]string(nil), cfg.OutputModules...),
|
||||||
IDsReassigned: built.IDsReassigned,
|
IDsReassigned: built.IDsReassigned,
|
||||||
SortingChangedInput: built.SortingChanged,
|
SortingChangedInput: built.SortingChanged,
|
||||||
SegmentsWithCategories: built.SegmentsWithCategories,
|
SegmentsWithCategories: built.SegmentsWithCategories,
|
||||||
|
TimingFieldsRepaired: parsed.Stats.TimingFieldsRepaired,
|
||||||
|
TimingOrderSwapped: parsed.Stats.TimingOrderSwapped,
|
||||||
|
SpeakerFilled: parsed.Stats.SpeakerFilled,
|
||||||
|
SegmentsDroppedText: parsed.Stats.SegmentsDroppedText,
|
||||||
}
|
}
|
||||||
auditJSON, err := json.Marshal(audit)
|
auditJSON, err := json.Marshal(audit)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -68,20 +78,24 @@ func Run(ctx context.Context, cfg config.NormalizeConfig) error {
|
|||||||
report.Info("normalize", "normalize", "started normalize command"),
|
report.Info("normalize", "normalize", "started normalize command"),
|
||||||
report.Info("normalize", "normalize", fmt.Sprintf("input file: %s", cfg.InputFile)),
|
report.Info("normalize", "normalize", fmt.Sprintf("input file: %s", cfg.InputFile)),
|
||||||
report.Info("normalize", "normalize", fmt.Sprintf("detected input shape: %s", parsed.Shape)),
|
report.Info("normalize", "normalize", fmt.Sprintf("detected input shape: %s", parsed.Shape)),
|
||||||
report.Info("normalize", "normalize", fmt.Sprintf("input segment count: %d", len(parsed.Segments))),
|
report.Info("normalize", "normalize", fmt.Sprintf("input segment count: %d", parsed.InputSegmentCount)),
|
||||||
report.Info("normalize", "normalize", fmt.Sprintf("selected output schema: %s", cfg.OutputSchema)),
|
report.Info("normalize", "normalize", fmt.Sprintf("selected output schema: %s", cfg.OutputSchema)),
|
||||||
report.Info("normalize", "normalize", fmt.Sprintf("selected output modules: %s", strings.Join(cfg.OutputModules, ","))),
|
report.Info("normalize", "normalize", fmt.Sprintf("selected output modules: %s", strings.Join(cfg.OutputModules, ","))),
|
||||||
report.Info("normalize", "normalize", fmt.Sprintf("output file: %s", cfg.OutputFile)),
|
report.Info("normalize", "normalize", fmt.Sprintf("output file: %s", cfg.OutputFile)),
|
||||||
report.Info("normalize", "normalize", fmt.Sprintf("ids reassigned: %t", built.IDsReassigned)),
|
report.Info("normalize", "normalize", fmt.Sprintf("ids reassigned: %t", built.IDsReassigned)),
|
||||||
report.Info("normalize", "normalize", fmt.Sprintf("sorting changed input order: %t", built.SortingChanged)),
|
report.Info("normalize", "normalize", fmt.Sprintf("sorting changed input order: %t", built.SortingChanged)),
|
||||||
report.Info("normalize", "normalize", fmt.Sprintf("segments with categories: %d", built.SegmentsWithCategories)),
|
report.Info("normalize", "normalize", fmt.Sprintf("segments with categories: %d", built.SegmentsWithCategories)),
|
||||||
|
report.Info("normalize", "normalize", fmt.Sprintf("timing fields repaired: %d", parsed.Stats.TimingFieldsRepaired)),
|
||||||
|
report.Info("normalize", "normalize", fmt.Sprintf("timing order swapped: %d", parsed.Stats.TimingOrderSwapped)),
|
||||||
|
report.Info("normalize", "normalize", fmt.Sprintf("speaker placeholders added: %d", parsed.Stats.SpeakerFilled)),
|
||||||
|
report.Info("normalize", "normalize", fmt.Sprintf("segments dropped for empty text: %d", parsed.Stats.SegmentsDroppedText)),
|
||||||
report.Info("normalize", "normalize-audit", string(auditJSON)),
|
report.Info("normalize", "normalize-audit", string(auditJSON)),
|
||||||
}
|
}
|
||||||
if len(parsed.Segments) == 0 {
|
if parsed.InputSegmentCount == 0 {
|
||||||
events = append(events, report.Warning("normalize", "normalize", "input transcript contains zero segments"))
|
events = append(events, report.Warning("normalize", "normalize", "input transcript contains zero segments"))
|
||||||
}
|
}
|
||||||
events = append(events,
|
events = append(events,
|
||||||
report.Info("normalize", "validate-output", fmt.Sprintf("validated %d output segment(s)", len(parsed.Segments))),
|
report.Info("normalize", "validate-output", fmt.Sprintf("validated %d output segment(s)", built.OutputSegmentCount)),
|
||||||
report.Info("output", "json", "wrote transcript JSON"),
|
report.Info("output", "json", "wrote transcript JSON"),
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|||||||
@@ -19,14 +19,20 @@ const (
|
|||||||
|
|
||||||
// ParsedTranscript is the validated normalize input model.
|
// ParsedTranscript is the validated normalize input model.
|
||||||
type ParsedTranscript struct {
|
type ParsedTranscript struct {
|
||||||
Shape InputShape
|
Shape InputShape
|
||||||
Segments []InputSegment
|
InputSegmentCount int
|
||||||
|
Stats NormalizeStats
|
||||||
|
Segments []InputSegment
|
||||||
}
|
}
|
||||||
|
|
||||||
// InputSegment is a validated segment from normalize input.
|
// InputSegment is a validated segment from normalize input.
|
||||||
type InputSegment struct {
|
type InputSegment struct {
|
||||||
InputIndex int
|
InputIndex int
|
||||||
OriginalID *int
|
OriginalID *int
|
||||||
|
StartPresent bool
|
||||||
|
EndPresent bool
|
||||||
|
SpeakerPresent bool
|
||||||
|
TextPresent bool
|
||||||
Start float64
|
Start float64
|
||||||
End float64
|
End float64
|
||||||
Speaker string
|
Speaker string
|
||||||
@@ -39,6 +45,14 @@ type InputSegment struct {
|
|||||||
OverlapGroupID *int
|
OverlapGroupID *int
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NormalizeStats captures deterministic repair/drop outcomes from normalize input processing.
|
||||||
|
type NormalizeStats struct {
|
||||||
|
TimingFieldsRepaired int
|
||||||
|
TimingOrderSwapped int
|
||||||
|
SpeakerFilled int
|
||||||
|
SegmentsDroppedText int
|
||||||
|
}
|
||||||
|
|
||||||
type inputSegmentPayload struct {
|
type inputSegmentPayload struct {
|
||||||
ID *int `json:"id"`
|
ID *int `json:"id"`
|
||||||
Start *float64 `json:"start"`
|
Start *float64 `json:"start"`
|
||||||
@@ -85,13 +99,15 @@ func ParseReader(reader io.Reader) (ParsedTranscript, error) {
|
|||||||
case '{':
|
case '{':
|
||||||
return parseObjectShape(trimmed)
|
return parseObjectShape(trimmed)
|
||||||
case '[':
|
case '[':
|
||||||
segments, err := parseSegmentsArray(trimmed)
|
segments, stats, inputCount, err := parseSegmentsArray(trimmed)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ParsedTranscript{}, err
|
return ParsedTranscript{}, err
|
||||||
}
|
}
|
||||||
return ParsedTranscript{
|
return ParsedTranscript{
|
||||||
Shape: ShapeBareSegmentsArray,
|
Shape: ShapeBareSegmentsArray,
|
||||||
Segments: segments,
|
InputSegmentCount: inputCount,
|
||||||
|
Stats: stats,
|
||||||
|
Segments: segments,
|
||||||
}, nil
|
}, nil
|
||||||
default:
|
default:
|
||||||
return ParsedTranscript{}, fmt.Errorf("normalize input must be a top-level object with \"segments\" or a top-level segment array")
|
return ParsedTranscript{}, fmt.Errorf("normalize input must be a top-level object with \"segments\" or a top-level segment array")
|
||||||
@@ -121,72 +137,73 @@ func parseObjectShape(raw []byte) (ParsedTranscript, error) {
|
|||||||
return ParsedTranscript{}, fmt.Errorf("normalize object input must contain a \"segments\" field")
|
return ParsedTranscript{}, fmt.Errorf("normalize object input must contain a \"segments\" field")
|
||||||
}
|
}
|
||||||
|
|
||||||
segments, err := parseSegmentsArray(segmentsRaw)
|
segments, stats, inputCount, err := parseSegmentsArray(segmentsRaw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ParsedTranscript{}, err
|
return ParsedTranscript{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
return ParsedTranscript{
|
return ParsedTranscript{
|
||||||
Shape: ShapeObjectWithSegments,
|
Shape: ShapeObjectWithSegments,
|
||||||
Segments: segments,
|
InputSegmentCount: inputCount,
|
||||||
|
Stats: stats,
|
||||||
|
Segments: segments,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func parseSegmentsArray(raw []byte) ([]InputSegment, error) {
|
func parseSegmentsArray(raw []byte) ([]InputSegment, NormalizeStats, int, error) {
|
||||||
var segmentValues []json.RawMessage
|
var segmentValues []json.RawMessage
|
||||||
if err := json.Unmarshal(raw, &segmentValues); err != nil {
|
if err := json.Unmarshal(raw, &segmentValues); err != nil {
|
||||||
return nil, fmt.Errorf("normalize input \"segments\" must be an array")
|
return nil, NormalizeStats{}, 0, fmt.Errorf("normalize input \"segments\" must be an array")
|
||||||
}
|
}
|
||||||
|
|
||||||
segments := make([]InputSegment, len(segmentValues))
|
segments := make([]InputSegment, len(segmentValues))
|
||||||
for index, segmentRaw := range segmentValues {
|
for index, segmentRaw := range segmentValues {
|
||||||
segment, err := parseSegment(index, segmentRaw)
|
segment, err := decodeSegment(index, segmentRaw)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, NormalizeStats{}, 0, err
|
||||||
}
|
}
|
||||||
segments[index] = segment
|
segments[index] = segment
|
||||||
}
|
}
|
||||||
return segments, nil
|
normalized, stats, err := normalizeSegments(segments)
|
||||||
|
return normalized, stats, len(segmentValues), err
|
||||||
}
|
}
|
||||||
|
|
||||||
func parseSegment(index int, raw []byte) (InputSegment, error) {
|
func decodeSegment(index int, raw []byte) (InputSegment, error) {
|
||||||
var payload inputSegmentPayload
|
var payload inputSegmentPayload
|
||||||
if err := json.Unmarshal(raw, &payload); err != nil {
|
if err := json.Unmarshal(raw, &payload); err != nil {
|
||||||
return InputSegment{}, fmt.Errorf("segment %d: invalid segment object: %w", index, err)
|
return InputSegment{}, fmt.Errorf("segment %d: invalid segment object: %w", index, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if payload.Start == nil {
|
var start float64
|
||||||
return InputSegment{}, fmt.Errorf("segment %d is missing required field \"start\"", index)
|
if payload.Start != nil {
|
||||||
|
start = *payload.Start
|
||||||
}
|
}
|
||||||
if payload.End == nil {
|
var end float64
|
||||||
return InputSegment{}, fmt.Errorf("segment %d is missing required field \"end\"", index)
|
if payload.End != nil {
|
||||||
}
|
end = *payload.End
|
||||||
if payload.Speaker == nil {
|
|
||||||
return InputSegment{}, fmt.Errorf("segment %d is missing required field \"speaker\"", index)
|
|
||||||
}
|
|
||||||
if payload.Text == nil {
|
|
||||||
return InputSegment{}, fmt.Errorf("segment %d is missing required field \"text\"", index)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if *payload.Start < 0 {
|
speaker := ""
|
||||||
return InputSegment{}, fmt.Errorf("segment %d has start %v; start must be >= 0", index, *payload.Start)
|
if payload.Speaker != nil {
|
||||||
}
|
speaker = strings.TrimSpace(*payload.Speaker)
|
||||||
if *payload.End < *payload.Start {
|
|
||||||
return InputSegment{}, fmt.Errorf("segment %d has end %v before start %v", index, *payload.End, *payload.Start)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
speaker := strings.TrimSpace(*payload.Speaker)
|
text := ""
|
||||||
if speaker == "" {
|
if payload.Text != nil {
|
||||||
return InputSegment{}, fmt.Errorf("segment %d has empty \"speaker\"; speaker must be non-empty", index)
|
text = *payload.Text
|
||||||
}
|
}
|
||||||
|
|
||||||
return InputSegment{
|
return InputSegment{
|
||||||
InputIndex: index,
|
InputIndex: index,
|
||||||
OriginalID: payload.ID,
|
OriginalID: payload.ID,
|
||||||
Start: *payload.Start,
|
StartPresent: payload.Start != nil,
|
||||||
End: *payload.End,
|
EndPresent: payload.End != nil,
|
||||||
|
SpeakerPresent: payload.Speaker != nil,
|
||||||
|
TextPresent: payload.Text != nil,
|
||||||
|
Start: start,
|
||||||
|
End: end,
|
||||||
Speaker: speaker,
|
Speaker: speaker,
|
||||||
Text: *payload.Text,
|
Text: text,
|
||||||
Categories: append([]string(nil), payload.Categories...),
|
Categories: append([]string(nil), payload.Categories...),
|
||||||
Source: payload.Source,
|
Source: payload.Source,
|
||||||
SourceSegmentIndex: payload.SourceSegmentIndex,
|
SourceSegmentIndex: payload.SourceSegmentIndex,
|
||||||
@@ -195,3 +212,96 @@ func parseSegment(index int, raw []byte) (InputSegment, error) {
|
|||||||
OverlapGroupID: payload.OverlapGroupID,
|
OverlapGroupID: payload.OverlapGroupID,
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func normalizeSegments(segments []InputSegment) ([]InputSegment, NormalizeStats, error) {
|
||||||
|
stats := NormalizeStats{}
|
||||||
|
filtered := make([]InputSegment, 0, len(segments))
|
||||||
|
for _, segment := range segments {
|
||||||
|
if strings.TrimSpace(segment.Text) == "" {
|
||||||
|
stats.SegmentsDroppedText++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
filtered = append(filtered, segment)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(filtered) == 0 {
|
||||||
|
return filtered, stats, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
hasStart := make([]bool, len(filtered))
|
||||||
|
hasEnd := make([]bool, len(filtered))
|
||||||
|
for index, segment := range filtered {
|
||||||
|
hasStart[index] = segment.StartPresent
|
||||||
|
hasEnd[index] = segment.EndPresent
|
||||||
|
}
|
||||||
|
for index := range filtered {
|
||||||
|
if !hasStart[index] && !hasEnd[index] {
|
||||||
|
midpoint := inferTimestampFromNeighbors(filtered, hasStart, hasEnd, index)
|
||||||
|
filtered[index].Start = midpoint
|
||||||
|
filtered[index].End = midpoint
|
||||||
|
stats.TimingFieldsRepaired += 2
|
||||||
|
hasStart[index] = true
|
||||||
|
hasEnd[index] = true
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if hasStart[index] && !hasEnd[index] {
|
||||||
|
filtered[index].End = filtered[index].Start
|
||||||
|
stats.TimingFieldsRepaired++
|
||||||
|
hasEnd[index] = true
|
||||||
|
}
|
||||||
|
if !hasStart[index] && hasEnd[index] {
|
||||||
|
filtered[index].Start = filtered[index].End
|
||||||
|
stats.TimingFieldsRepaired++
|
||||||
|
hasStart[index] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for index := range filtered {
|
||||||
|
if filtered[index].Start < 0 {
|
||||||
|
return nil, NormalizeStats{}, fmt.Errorf("segment %d has start %v; start must be >= 0", filtered[index].InputIndex, filtered[index].Start)
|
||||||
|
}
|
||||||
|
if filtered[index].End < filtered[index].Start {
|
||||||
|
filtered[index].Start, filtered[index].End = filtered[index].End, filtered[index].Start
|
||||||
|
stats.TimingOrderSwapped++
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(filtered[index].Speaker) == "" {
|
||||||
|
filtered[index].Speaker = "Unknown_Speaker"
|
||||||
|
stats.SpeakerFilled++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return filtered, stats, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func inferTimestampFromNeighbors(segments []InputSegment, hasStart []bool, hasEnd []bool, index int) float64 {
|
||||||
|
prevEnd, hasPrev := nearestPreviousEnd(segments, hasEnd, index)
|
||||||
|
nextStart, hasNext := nearestNextStart(segments, hasStart, index)
|
||||||
|
if hasPrev && hasNext {
|
||||||
|
return (prevEnd + nextStart) / 2
|
||||||
|
}
|
||||||
|
if hasPrev {
|
||||||
|
return prevEnd
|
||||||
|
}
|
||||||
|
if hasNext {
|
||||||
|
return nextStart
|
||||||
|
}
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
func nearestPreviousEnd(segments []InputSegment, hasEnd []bool, index int) (float64, bool) {
|
||||||
|
for i := index - 1; i >= 0; i-- {
|
||||||
|
if hasEnd[i] {
|
||||||
|
return segments[i].End, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
|
||||||
|
func nearestNextStart(segments []InputSegment, hasStart []bool, index int) (float64, bool) {
|
||||||
|
for i := index + 1; i < len(segments); i++ {
|
||||||
|
if hasStart[i] {
|
||||||
|
return segments[i].Start, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return 0, false
|
||||||
|
}
|
||||||
|
|||||||
@@ -92,34 +92,70 @@ func TestParseReaderTopLevelScalarShapesFail(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseReaderMissingStartFails(t *testing.T) {
|
func TestParseReaderMissingStartUsesEndValue(t *testing.T) {
|
||||||
_, err := ParseReader(strings.NewReader(`[{"end":2,"speaker":"A","text":"t"}]`))
|
parsed, err := ParseReader(strings.NewReader(`[{"end":2,"speaker":"A","text":"t"}]`))
|
||||||
assertContains(t, err, `missing required field "start"`)
|
if err != nil {
|
||||||
|
t.Fatalf("parse failed: %v", err)
|
||||||
|
}
|
||||||
|
if len(parsed.Segments) != 1 {
|
||||||
|
t.Fatalf("segment count = %d, want 1", len(parsed.Segments))
|
||||||
|
}
|
||||||
|
if parsed.Segments[0].Start != 2 || parsed.Segments[0].End != 2 {
|
||||||
|
t.Fatalf("segment timing = %v..%v, want 2..2", parsed.Segments[0].Start, parsed.Segments[0].End)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseReaderMissingEndFails(t *testing.T) {
|
func TestParseReaderMissingEndUsesStartValue(t *testing.T) {
|
||||||
_, err := ParseReader(strings.NewReader(`[{"start":1,"speaker":"A","text":"t"}]`))
|
parsed, err := ParseReader(strings.NewReader(`[{"start":1,"speaker":"A","text":"t"}]`))
|
||||||
assertContains(t, err, `missing required field "end"`)
|
if err != nil {
|
||||||
|
t.Fatalf("parse failed: %v", err)
|
||||||
|
}
|
||||||
|
if len(parsed.Segments) != 1 {
|
||||||
|
t.Fatalf("segment count = %d, want 1", len(parsed.Segments))
|
||||||
|
}
|
||||||
|
if parsed.Segments[0].Start != 1 || parsed.Segments[0].End != 1 {
|
||||||
|
t.Fatalf("segment timing = %v..%v, want 1..1", parsed.Segments[0].Start, parsed.Segments[0].End)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseReaderMissingSpeakerFails(t *testing.T) {
|
func TestParseReaderMissingSpeakerUsesUnknownPlaceholder(t *testing.T) {
|
||||||
_, err := ParseReader(strings.NewReader(`[{"start":1,"end":2,"text":"t"}]`))
|
parsed, err := ParseReader(strings.NewReader(`[{"start":1,"end":2,"text":"t"}]`))
|
||||||
assertContains(t, err, `missing required field "speaker"`)
|
if err != nil {
|
||||||
|
t.Fatalf("parse failed: %v", err)
|
||||||
|
}
|
||||||
|
if parsed.Segments[0].Speaker != "Unknown_Speaker" {
|
||||||
|
t.Fatalf("speaker = %q, want Unknown_Speaker", parsed.Segments[0].Speaker)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseReaderEmptySpeakerFails(t *testing.T) {
|
func TestParseReaderEmptySpeakerUsesUnknownPlaceholder(t *testing.T) {
|
||||||
_, err := ParseReader(strings.NewReader(`[{"start":1,"end":2,"speaker":" ","text":"t"}]`))
|
parsed, err := ParseReader(strings.NewReader(`[{"start":1,"end":2,"speaker":" ","text":"t"}]`))
|
||||||
assertContains(t, err, `speaker must be non-empty`)
|
if err != nil {
|
||||||
|
t.Fatalf("parse failed: %v", err)
|
||||||
|
}
|
||||||
|
if parsed.Segments[0].Speaker != "Unknown_Speaker" {
|
||||||
|
t.Fatalf("speaker = %q, want Unknown_Speaker", parsed.Segments[0].Speaker)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseReaderMissingTextFails(t *testing.T) {
|
func TestParseReaderMissingTextDropsSegment(t *testing.T) {
|
||||||
_, err := ParseReader(strings.NewReader(`[{"start":1,"end":2,"speaker":"A"}]`))
|
parsed, err := ParseReader(strings.NewReader(`[{"start":1,"end":2,"speaker":"A"}]`))
|
||||||
assertContains(t, err, `missing required field "text"`)
|
if err != nil {
|
||||||
|
t.Fatalf("parse failed: %v", err)
|
||||||
|
}
|
||||||
|
if len(parsed.Segments) != 0 {
|
||||||
|
t.Fatalf("segment count = %d, want 0", len(parsed.Segments))
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseReaderEndBeforeStartFails(t *testing.T) {
|
func TestParseReaderEndBeforeStartSwapsValues(t *testing.T) {
|
||||||
_, err := ParseReader(strings.NewReader(`[{"start":3,"end":2,"speaker":"A","text":"t"}]`))
|
parsed, err := ParseReader(strings.NewReader(`[{"start":3,"end":2,"speaker":"A","text":"t"}]`))
|
||||||
assertContains(t, err, "before start")
|
if err != nil {
|
||||||
|
t.Fatalf("parse failed: %v", err)
|
||||||
|
}
|
||||||
|
if parsed.Segments[0].Start != 2 || parsed.Segments[0].End != 3 {
|
||||||
|
t.Fatalf("segment timing = %v..%v, want 2..3", parsed.Segments[0].Start, parsed.Segments[0].End)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestParseReaderNegativeStartFails(t *testing.T) {
|
func TestParseReaderNegativeStartFails(t *testing.T) {
|
||||||
@@ -153,6 +189,68 @@ func TestParseReaderCategoriesPreservedWhenValid(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestParseReaderMissingBothTimesMiddleUsesNeighborMidpoint(t *testing.T) {
|
||||||
|
parsed, err := ParseReader(strings.NewReader(`[
|
||||||
|
{"start":1,"end":2,"speaker":"A","text":"left"},
|
||||||
|
{"speaker":"B","text":"middle"},
|
||||||
|
{"start":6,"end":7,"speaker":"C","text":"right"}
|
||||||
|
]`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse failed: %v", err)
|
||||||
|
}
|
||||||
|
if parsed.Segments[1].Start != 4 || parsed.Segments[1].End != 4 {
|
||||||
|
t.Fatalf("middle timing = %v..%v, want 4..4", parsed.Segments[1].Start, parsed.Segments[1].End)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseReaderMissingBothTimesEdgeFallbacks(t *testing.T) {
|
||||||
|
parsedFirst, err := ParseReader(strings.NewReader(`[
|
||||||
|
{"speaker":"A","text":"first"},
|
||||||
|
{"start":5,"end":6,"speaker":"B","text":"second"}
|
||||||
|
]`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse first failed: %v", err)
|
||||||
|
}
|
||||||
|
if parsedFirst.Segments[0].Start != 5 || parsedFirst.Segments[0].End != 5 {
|
||||||
|
t.Fatalf("first timing = %v..%v, want 5..5", parsedFirst.Segments[0].Start, parsedFirst.Segments[0].End)
|
||||||
|
}
|
||||||
|
|
||||||
|
parsedLast, err := ParseReader(strings.NewReader(`[
|
||||||
|
{"start":1,"end":2,"speaker":"A","text":"first"},
|
||||||
|
{"speaker":"B","text":"last"}
|
||||||
|
]`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse last failed: %v", err)
|
||||||
|
}
|
||||||
|
if parsedLast.Segments[1].Start != 2 || parsedLast.Segments[1].End != 2 {
|
||||||
|
t.Fatalf("last timing = %v..%v, want 2..2", parsedLast.Segments[1].Start, parsedLast.Segments[1].End)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseReaderMissingBothTimesSingleSegmentUsesZero(t *testing.T) {
|
||||||
|
parsed, err := ParseReader(strings.NewReader(`[{"speaker":"A","text":"only"}]`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse failed: %v", err)
|
||||||
|
}
|
||||||
|
if parsed.Segments[0].Start != 0 || parsed.Segments[0].End != 0 {
|
||||||
|
t.Fatalf("timing = %v..%v, want 0..0", parsed.Segments[0].Start, parsed.Segments[0].End)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseReaderEmptyOrWhitespaceTextDropped(t *testing.T) {
|
||||||
|
parsed, err := ParseReader(strings.NewReader(`[
|
||||||
|
{"start":1,"end":2,"speaker":"A","text":"ok"},
|
||||||
|
{"start":2,"end":3,"speaker":"A","text":""},
|
||||||
|
{"start":3,"end":4,"speaker":"A","text":" "}
|
||||||
|
]`))
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse failed: %v", err)
|
||||||
|
}
|
||||||
|
if len(parsed.Segments) != 1 {
|
||||||
|
t.Fatalf("segment count = %d, want 1", len(parsed.Segments))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestParseReaderOriginalInputIndexPreserved(t *testing.T) {
|
func TestParseReaderOriginalInputIndexPreserved(t *testing.T) {
|
||||||
input := `[
|
input := `[
|
||||||
{"start":1,"end":2,"speaker":"A","text":"one"},
|
{"start":1,"end":2,"speaker":"A","text":"one"},
|
||||||
|
|||||||
Reference in New Issue
Block a user