From 0b1b670baf86d2d23ed1b7b1dd2d1c7458121d6d Mon Sep 17 00:00:00 2001 From: Eric Rakestraw Date: Mon, 11 May 2026 00:12:19 +0000 Subject: [PATCH] Add deterministic transcript normalization --- internal/core/normalization/normalize.go | 180 ++++++++++ internal/core/normalization/normalize_test.go | 309 ++++++++++++++++++ internal/core/normalization/tokens.go | 33 ++ 3 files changed, 522 insertions(+) create mode 100644 internal/core/normalization/normalize.go create mode 100644 internal/core/normalization/normalize_test.go create mode 100644 internal/core/normalization/tokens.go diff --git a/internal/core/normalization/normalize.go b/internal/core/normalization/normalize.go new file mode 100644 index 0000000..392b0c5 --- /dev/null +++ b/internal/core/normalization/normalize.go @@ -0,0 +1,180 @@ +package normalization + +import ( + "sort" + "strings" + + "gitea.maximumdirect.net/eric/audita/internal/core/schema" +) + +// NormalizationConfig holds configuration parameters for transcript normalization +type NormalizationConfig struct { + MaxSegmentGap float64 // Maximum gap between segments to consider for merging (seconds) + EllipsisGap float64 // Gap threshold above which to insert ellipsis (seconds) + MaxSegmentDuration float64 // Maximum duration for a merged segment (seconds) + MaxSegmentTokens int // Maximum token estimate for a merged segment +} + +// NormalizationSummary records the results and statistics of normalization +type NormalizationSummary struct { + InputSegmentCount int `json:"input_segment_count"` + OutputSegmentCount int `json:"output_segment_count"` + MergesPerformed int `json:"merges_performed"` + IDsReassigned int `json:"ids_reassigned"` + SkippedMerges struct { + DifferentSpeakers int `json:"different_speakers"` + GapTooLarge int `json:"gap_too_large"` + DurationExceeded int `json:"duration_exceeded"` + TokenLimitExceeded int `json:"token_limit_exceeded"` + } `json:"skipped_merges"` +} + +// NormalizeTranscript performs deterministic normalization on a transcript +type NormalizeTranscript struct { + config NormalizationConfig + estimator *SimpleTokenEstimator +} + +// NewNormalizer creates a new transcript normalizer with the given configuration +func NewNormalizer(config NormalizationConfig) *NormalizeTranscript { + return &NormalizeTranscript{ + config: config, + estimator: &SimpleTokenEstimator{}, + } +} + +// Normalize performs deterministic normalization on the given transcript +func (n *NormalizeTranscript) Normalize(transcript *schema.Transcript) (*schema.Transcript, *NormalizationSummary) { + summary := &NormalizationSummary{ + InputSegmentCount: len(transcript.Segments), + } + + if len(transcript.Segments) == 0 { + return transcript, summary + } + + // Sort segments chronologically + sortedSegments := make([]schema.Segment, len(transcript.Segments)) + copy(sortedSegments, transcript.Segments) + sort.Slice(sortedSegments, func(i, j int) bool { + return sortedSegments[i].Start < sortedSegments[j].Start + }) + + // Merge same-speaker adjacent segments + var normalizedSegments []schema.Segment + var currentSegment schema.Segment + var sourceIDs []int + + for i, segment := range sortedSegments { + if i == 0 { + // Initialize with first segment + currentSegment = segment + sourceIDs = []int{segment.ID} + continue + } + + // Check if we should merge with current segment + gap := segment.Start - currentSegment.End + merge, reason := shouldMergeWithReason(n.config, n.estimator, ¤tSegment, &segment, gap) + if merge { + summary.MergesPerformed++ + currentSegment = mergeSegments(¤tSegment, &segment, gap, n.config.EllipsisGap) + sourceIDs = append(sourceIDs, segment.ID) + } else { + // Track the reason for not merging + switch reason { + case "different_speakers": + summary.SkippedMerges.DifferentSpeakers++ + case "gap_too_large": + summary.SkippedMerges.GapTooLarge++ + case "duration_exceeded": + summary.SkippedMerges.DurationExceeded++ + case "token_limit_exceeded": + summary.SkippedMerges.TokenLimitExceeded++ + } + // Finalize current segment and start new one + normalizedSegments = append(normalizedSegments, currentSegment) + currentSegment = segment + sourceIDs = []int{segment.ID} + } + } + + // Add the last segment + if len(currentSegment.Text) > 0 { + normalizedSegments = append(normalizedSegments, currentSegment) + } + + // Reassign sequential IDs starting at 1 + for i, segment := range normalizedSegments { + if segment.ID != i+1 { + summary.IDsReassigned++ + } + normalizedSegments[i].ID = i + 1 + } + + summary.OutputSegmentCount = len(normalizedSegments) + + return &schema.Transcript{Segments: normalizedSegments}, summary +} + +func shouldMerge(config NormalizationConfig, estimator *SimpleTokenEstimator, + current, next *schema.Segment, gap float64) bool { + + // Don't merge if different speakers + if current.Speaker != next.Speaker { + return false + } + + // Don't merge if gap is too large + if gap > config.MaxSegmentGap { + return false + } + + // Calculate what the merged segment would look like + mergedText := current.Text + if gap >= config.EllipsisGap { + mergedText += "... " + } else { + mergedText += " " + } + mergedText += next.Text + + mergedDuration := next.End - current.Start + + // Don't merge if duration would be exceeded + if mergedDuration > config.MaxSegmentDuration { + return false + } + + // Don't merge if token estimate would be exceeded + tokenEstimate := estimator.EstimateTokens(mergedText) + if tokenEstimate > config.MaxSegmentTokens { + return false + } + + return true +} + +func mergeSegments(current, next *schema.Segment, gap, ellipsisGap float64) schema.Segment { + mergedText := current.Text + if gap >= ellipsisGap { + mergedText += "... " + } else { + mergedText += " " + } + mergedText += next.Text + + // Merge categories + categories := make([]string, 0, len(current.Categories)+len(next.Categories)) + categories = append(categories, current.Categories...) + categories = append(categories, next.Categories...) + + return schema.Segment{ + ID: current.ID, // Will be reassigned later + Speaker: current.Speaker, + Start: current.Start, + End: next.End, + Text: mergedText, + Categories: categories, + } +} diff --git a/internal/core/normalization/normalize_test.go b/internal/core/normalization/normalize_test.go new file mode 100644 index 0000000..4280f4a --- /dev/null +++ b/internal/core/normalization/normalize_test.go @@ -0,0 +1,309 @@ +package normalization + +import ( + "testing" + + "gitea.maximumdirect.net/eric/audita/internal/core/schema" +) + +func TestNormalizationNoOp(t *testing.T) { + config := NormalizationConfig{ + MaxSegmentGap: 2.0, + EllipsisGap: 1.0, + MaxSegmentDuration: 60.0, + MaxSegmentTokens: 100, + } + normalizer := NewNormalizer(config) + + transcript := &schema.Transcript{ + Segments: []schema.Segment{ + {ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"}, + {ID: 2, Speaker: "Bob", Start: 2.0, End: 3.0, Text: "Hi"}, + }, + } + + normalized, summary := normalizer.Normalize(transcript) + + // Should be no-op since different speakers + if len(normalized.Segments) != 2 { + t.Errorf("expected 2 segments, got %d", len(normalized.Segments)) + } + if summary.InputSegmentCount != 2 { + t.Errorf("expected input count 2, got %d", summary.InputSegmentCount) + } + if summary.OutputSegmentCount != 2 { + t.Errorf("expected output count 2, got %d", summary.OutputSegmentCount) + } + if summary.MergesPerformed != 0 { + t.Errorf("expected 0 merges, got %d", summary.MergesPerformed) + } +} + +func TestNormalizationSameSpeakerMergeWithinGap(t *testing.T) { + config := NormalizationConfig{ + MaxSegmentGap: 2.0, + EllipsisGap: 1.0, + MaxSegmentDuration: 60.0, + MaxSegmentTokens: 100, + } + normalizer := NewNormalizer(config) + + transcript := &schema.Transcript{ + Segments: []schema.Segment{ + {ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"}, + {ID: 2, Speaker: "Alice", Start: 1.5, End: 2.5, Text: "world"}, + }, + } + + normalized, summary := normalizer.Normalize(transcript) + + // Should merge since same speaker and small gap + if len(normalized.Segments) != 1 { + t.Errorf("expected 1 merged segment, got %d", len(normalized.Segments)) + } + if summary.MergesPerformed != 1 { + t.Errorf("expected 1 merge, got %d", summary.MergesPerformed) + } + if !strings.Contains(normalized.Segments[0].Text, "Hello") || !strings.Contains(normalized.Segments[0].Text, "world") { + t.Errorf("expected merged text to contain both parts, got %q", normalized.Segments[0].Text) + } +} + +func TestNormalizationEllipsisInsertionAboveThreshold(t *testing.T) { + config := NormalizationConfig{ + MaxSegmentGap: 2.0, + EllipsisGap: 0.5, + MaxSegmentDuration: 60.0, + MaxSegmentTokens: 100, + } + normalizer := NewNormalizer(config) + + transcript := &schema.Transcript{ + Segments: []schema.Segment{ + {ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"}, + {ID: 2, Speaker: "Alice", Start: 2.0, End: 3.0, Text: "world"}, + }, + } + + normalized, _ := normalizer.Normalize(transcript) + + // Should merge with ellipsis since gap (1.0) > ellipsis threshold (0.5) + if len(normalized.Segments) != 1 { + t.Errorf("expected 1 merged segment, got %d", len(normalized.Segments)) + } + if !strings.Contains(normalized.Segments[0].Text, "...") { + t.Errorf("expected ellipsis in merged text, got %q", normalized.Segments[0].Text) + } +} + +func TestNormalizationNoMergeDifferentSpeakers(t *testing.T) { + config := NormalizationConfig{ + MaxSegmentGap: 2.0, + EllipsisGap: 1.0, + MaxSegmentDuration: 60.0, + MaxSegmentTokens: 100, + } + normalizer := NewNormalizer(config) + + transcript := &schema.Transcript{ + Segments: []schema.Segment{ + {ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"}, + {ID: 2, Speaker: "Bob", Start: 1.5, End: 2.5, Text: "Hi"}, + }, + } + + normalized, summary := normalizer.Normalize(transcript) + + // Should NOT merge since different speakers + if len(normalized.Segments) != 2 { + t.Errorf("expected 2 segments, got %d", len(normalized.Segments)) + } + if summary.SkippedMerges.DifferentSpeakers != 1 { + t.Errorf("expected 1 skipped merge for different speakers, got %d", summary.SkippedMerges.DifferentSpeakers) + } +} + +func TestNormalizationNoMergeWhenMaxDurationExceeded(t *testing.T) { + config := NormalizationConfig{ + MaxSegmentGap: 2.0, + EllipsisGap: 1.0, + MaxSegmentDuration: 1.0, // Very small max duration + MaxSegmentTokens: 100, + } + normalizer := NewNormalizer(config) + + transcript := &schema.Transcript{ + Segments: []schema.Segment{ + {ID: 1, Speaker: "Alice", Start: 0.0, End: 0.5, Text: "Hello"}, + {ID: 2, Speaker: "Alice", Start: 0.6, End: 1.5, Text: "world"}, + }, + } + + normalized, summary := normalizer.Normalize(transcript) + + // Should NOT merge since merged duration (1.5) > max duration (1.0) + if len(normalized.Segments) != 2 { + t.Errorf("expected 2 segments, got %d", len(normalized.Segments)) + } + if summary.SkippedMerges.DurationExceeded != 1 { + t.Errorf("expected 1 skipped merge for duration exceeded, got %d", summary.SkippedMerges.DurationExceeded) + } +} + +func TestNormalizationNoMergeWhenTokenLimitExceeded(t *testing.T) { + config := NormalizationConfig{ + MaxSegmentGap: 2.0, + EllipsisGap: 1.0, + MaxSegmentDuration: 60.0, + MaxSegmentTokens: 3, // Very small token limit + } + normalizer := NewNormalizer(config) + + transcript := &schema.Transcript{ + Segments: []schema.Segment{ + {ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"}, + {ID: 2, Speaker: "Alice", Start: 1.5, End: 2.5, Text: "world"}, + }, + } + + normalized, summary := normalizer.Normalize(transcript) + + // Should NOT merge since merged token count would exceed limit + if len(normalized.Segments) != 2 { + t.Errorf("expected 2 segments, got %d", len(normalized.Segments)) + } + if summary.SkippedMerges.TokenLimitExceeded != 1 { + t.Errorf("expected 1 skipped merge for token limit exceeded, got %d", summary.SkippedMerges.TokenLimitExceeded) + } +} + +func TestNormalizationChronologicalOrdering(t *testing.T) { + config := NormalizationConfig{ + MaxSegmentGap: 2.0, + EllipsisGap: 1.0, + MaxSegmentDuration: 60.0, + MaxSegmentTokens: 100, + } + normalizer := NewNormalizer(config) + + transcript := &schema.Transcript{ + Segments: []schema.Segment{ + {ID: 2, Speaker: "Alice", Start: 5.0, End: 6.0, Text: "Later"}, + {ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "First"}, + {ID: 3, Speaker: "Alice", Start: 2.0, End: 3.0, Text: "Middle"}, + }, + } + + normalized, _ := normalizer.Normalize(transcript) + + // Should be sorted chronologically + if len(normalized.Segments) != 1 { + t.Errorf("expected 1 merged segment, got %d", len(normalized.Segments)) + } + if normalized.Segments[0].Start != 0.0 { + t.Errorf("expected first segment to start at 0.0, got %f", normalized.Segments[0].Start) + } + if normalized.Segments[0].End != 6.0 { + t.Errorf("expected merged segment to end at 6.0, got %f", normalized.Segments[0].End) + } +} + +func TestNormalizationSequentialIDReassignment(t *testing.T) { + config := NormalizationConfig{ + MaxSegmentGap: 2.0, + EllipsisGap: 1.0, + MaxSegmentDuration: 60.0, + MaxSegmentTokens: 100, + } + normalizer := NewNormalizer(config) + + transcript := &schema.Transcript{ + Segments: []schema.Segment{ + {ID: 5, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"}, + {ID: 10, Speaker: "Bob", Start: 2.0, End: 3.0, Text: "Hi"}, + }, + } + + normalized, summary := normalizer.Normalize(transcript) + + // Should have sequential IDs starting at 1 + if len(normalized.Segments) != 2 { + t.Errorf("expected 2 segments, got %d", len(normalized.Segments)) + } + if normalized.Segments[0].ID != 1 { + t.Errorf("expected first segment ID 1, got %d", normalized.Segments[0].ID) + } + if normalized.Segments[1].ID != 2 { + t.Errorf("expected second segment ID 2, got %d", normalized.Segments[1].ID) + } + if summary.IDsReassigned != 2 { + t.Errorf("expected 2 IDs reassigned, got %d", summary.IDsReassigned) + } +} + +func TestNormalizationCategoryPreservation(t *testing.T) { + config := NormalizationConfig{ + MaxSegmentGap: 2.0, + EllipsisGap: 1.0, + MaxSegmentDuration: 60.0, + MaxSegmentTokens: 100, + } + normalizer := NewNormalizer(config) + + transcript := &schema.Transcript{ + Segments: []schema.Segment{ + {ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello", Categories: []string{"greeting"}}, + {ID: 2, Speaker: "Alice", Start: 1.5, End: 2.5, Text: "world", Categories: []string{"response"}}, + }, + } + + normalized, _ := normalizer.Normalize(transcript) + + // Should merge and preserve both categories + if len(normalized.Segments) != 1 { + t.Errorf("expected 1 merged segment, got %d", len(normalized.Segments)) + } + if len(normalized.Segments[0].Categories) != 2 { + t.Errorf("expected 2 categories, got %d", len(normalized.Segments[0].Categories)) + } + if normalized.Segments[0].Categories[0] != "greeting" || normalized.Segments[0].Categories[1] != "response" { + t.Errorf("expected preserved categories, got %v", normalized.Segments[0].Categories) + } +} + +func TestNormalizationSummaryCounts(t *testing.T) { + config := NormalizationConfig{ + MaxSegmentGap: 2.0, + EllipsisGap: 1.0, + MaxSegmentDuration: 60.0, + MaxSegmentTokens: 100, + } + normalizer := NewNormalizer(config) + + transcript := &schema.Transcript{ + Segments: []schema.Segment{ + {ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"}, + {ID: 2, Speaker: "Alice", Start: 1.5, End: 2.5, Text: "world"}, + {ID: 3, Speaker: "Bob", Start: 3.0, End: 4.0, Text: "Hi"}, + }, + } + + _, summary := normalizer.Normalize(transcript) + + // Verify summary counts + if summary.InputSegmentCount != 3 { + t.Errorf("expected input count 3, got %d", summary.InputSegmentCount) + } + if summary.OutputSegmentCount != 2 { + t.Errorf("expected output count 2, got %d", summary.OutputSegmentCount) + } + if summary.MergesPerformed != 1 { + t.Errorf("expected 1 merge, got %d", summary.MergesPerformed) + } + if summary.IDsReassigned != 3 { + t.Errorf("expected 3 IDs reassigned, got %d", summary.IDsReassigned) + } + if summary.SkippedMerges.DifferentSpeakers != 1 { + t.Errorf("expected 1 skipped merge for different speakers, got %d", summary.SkippedMerges.DifferentSpeakers) + } +} diff --git a/internal/core/normalization/tokens.go b/internal/core/normalization/tokens.go new file mode 100644 index 0000000..8e69ed7 --- /dev/null +++ b/internal/core/normalization/tokens.go @@ -0,0 +1,33 @@ +package normalization + +import ( + "strings" + "unicode" +) + +// SimpleTokenEstimator provides a basic deterministic token estimation. +// This is a placeholder that can be replaced with a more sophisticated estimator later. +type SimpleTokenEstimator struct{} + +// EstimateTokens provides a rough estimate of the number of tokens in the given text. +// This implementation uses a simple heuristic: count words and punctuation as tokens. +func (e *SimpleTokenEstimator) EstimateTokens(text string) int { + if text == "" { + return 0 + } + + // Simple heuristic: split on whitespace and count non-empty segments + words := strings.Fields(text) + tokenCount := len(words) + + // Add some estimate for punctuation that might be separate tokens + punctuationCount := 0 + for _, r := range text { + if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' { + punctuationCount++ + } + } + + // Rough estimate: each word is a token, plus half the punctuation as separate tokens + return tokenCount + (punctuationCount / 2) +}