Add deterministic transcript normalization
This commit is contained in:
180
internal/core/normalization/normalize.go
Normal file
180
internal/core/normalization/normalize.go
Normal file
@@ -0,0 +1,180 @@
|
||||
package normalization
|
||||
|
||||
import (
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
)
|
||||
|
||||
// NormalizationConfig holds configuration parameters for transcript normalization
|
||||
type NormalizationConfig struct {
|
||||
MaxSegmentGap float64 // Maximum gap between segments to consider for merging (seconds)
|
||||
EllipsisGap float64 // Gap threshold above which to insert ellipsis (seconds)
|
||||
MaxSegmentDuration float64 // Maximum duration for a merged segment (seconds)
|
||||
MaxSegmentTokens int // Maximum token estimate for a merged segment
|
||||
}
|
||||
|
||||
// NormalizationSummary records the results and statistics of normalization
|
||||
type NormalizationSummary struct {
|
||||
InputSegmentCount int `json:"input_segment_count"`
|
||||
OutputSegmentCount int `json:"output_segment_count"`
|
||||
MergesPerformed int `json:"merges_performed"`
|
||||
IDsReassigned int `json:"ids_reassigned"`
|
||||
SkippedMerges struct {
|
||||
DifferentSpeakers int `json:"different_speakers"`
|
||||
GapTooLarge int `json:"gap_too_large"`
|
||||
DurationExceeded int `json:"duration_exceeded"`
|
||||
TokenLimitExceeded int `json:"token_limit_exceeded"`
|
||||
} `json:"skipped_merges"`
|
||||
}
|
||||
|
||||
// NormalizeTranscript performs deterministic normalization on a transcript
|
||||
type NormalizeTranscript struct {
|
||||
config NormalizationConfig
|
||||
estimator *SimpleTokenEstimator
|
||||
}
|
||||
|
||||
// NewNormalizer creates a new transcript normalizer with the given configuration
|
||||
func NewNormalizer(config NormalizationConfig) *NormalizeTranscript {
|
||||
return &NormalizeTranscript{
|
||||
config: config,
|
||||
estimator: &SimpleTokenEstimator{},
|
||||
}
|
||||
}
|
||||
|
||||
// Normalize performs deterministic normalization on the given transcript
|
||||
func (n *NormalizeTranscript) Normalize(transcript *schema.Transcript) (*schema.Transcript, *NormalizationSummary) {
|
||||
summary := &NormalizationSummary{
|
||||
InputSegmentCount: len(transcript.Segments),
|
||||
}
|
||||
|
||||
if len(transcript.Segments) == 0 {
|
||||
return transcript, summary
|
||||
}
|
||||
|
||||
// Sort segments chronologically
|
||||
sortedSegments := make([]schema.Segment, len(transcript.Segments))
|
||||
copy(sortedSegments, transcript.Segments)
|
||||
sort.Slice(sortedSegments, func(i, j int) bool {
|
||||
return sortedSegments[i].Start < sortedSegments[j].Start
|
||||
})
|
||||
|
||||
// Merge same-speaker adjacent segments
|
||||
var normalizedSegments []schema.Segment
|
||||
var currentSegment schema.Segment
|
||||
var sourceIDs []int
|
||||
|
||||
for i, segment := range sortedSegments {
|
||||
if i == 0 {
|
||||
// Initialize with first segment
|
||||
currentSegment = segment
|
||||
sourceIDs = []int{segment.ID}
|
||||
continue
|
||||
}
|
||||
|
||||
// Check if we should merge with current segment
|
||||
gap := segment.Start - currentSegment.End
|
||||
merge, reason := shouldMergeWithReason(n.config, n.estimator, ¤tSegment, &segment, gap)
|
||||
if merge {
|
||||
summary.MergesPerformed++
|
||||
currentSegment = mergeSegments(¤tSegment, &segment, gap, n.config.EllipsisGap)
|
||||
sourceIDs = append(sourceIDs, segment.ID)
|
||||
} else {
|
||||
// Track the reason for not merging
|
||||
switch reason {
|
||||
case "different_speakers":
|
||||
summary.SkippedMerges.DifferentSpeakers++
|
||||
case "gap_too_large":
|
||||
summary.SkippedMerges.GapTooLarge++
|
||||
case "duration_exceeded":
|
||||
summary.SkippedMerges.DurationExceeded++
|
||||
case "token_limit_exceeded":
|
||||
summary.SkippedMerges.TokenLimitExceeded++
|
||||
}
|
||||
// Finalize current segment and start new one
|
||||
normalizedSegments = append(normalizedSegments, currentSegment)
|
||||
currentSegment = segment
|
||||
sourceIDs = []int{segment.ID}
|
||||
}
|
||||
}
|
||||
|
||||
// Add the last segment
|
||||
if len(currentSegment.Text) > 0 {
|
||||
normalizedSegments = append(normalizedSegments, currentSegment)
|
||||
}
|
||||
|
||||
// Reassign sequential IDs starting at 1
|
||||
for i, segment := range normalizedSegments {
|
||||
if segment.ID != i+1 {
|
||||
summary.IDsReassigned++
|
||||
}
|
||||
normalizedSegments[i].ID = i + 1
|
||||
}
|
||||
|
||||
summary.OutputSegmentCount = len(normalizedSegments)
|
||||
|
||||
return &schema.Transcript{Segments: normalizedSegments}, summary
|
||||
}
|
||||
|
||||
func shouldMerge(config NormalizationConfig, estimator *SimpleTokenEstimator,
|
||||
current, next *schema.Segment, gap float64) bool {
|
||||
|
||||
// Don't merge if different speakers
|
||||
if current.Speaker != next.Speaker {
|
||||
return false
|
||||
}
|
||||
|
||||
// Don't merge if gap is too large
|
||||
if gap > config.MaxSegmentGap {
|
||||
return false
|
||||
}
|
||||
|
||||
// Calculate what the merged segment would look like
|
||||
mergedText := current.Text
|
||||
if gap >= config.EllipsisGap {
|
||||
mergedText += "... "
|
||||
} else {
|
||||
mergedText += " "
|
||||
}
|
||||
mergedText += next.Text
|
||||
|
||||
mergedDuration := next.End - current.Start
|
||||
|
||||
// Don't merge if duration would be exceeded
|
||||
if mergedDuration > config.MaxSegmentDuration {
|
||||
return false
|
||||
}
|
||||
|
||||
// Don't merge if token estimate would be exceeded
|
||||
tokenEstimate := estimator.EstimateTokens(mergedText)
|
||||
if tokenEstimate > config.MaxSegmentTokens {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
}
|
||||
|
||||
func mergeSegments(current, next *schema.Segment, gap, ellipsisGap float64) schema.Segment {
|
||||
mergedText := current.Text
|
||||
if gap >= ellipsisGap {
|
||||
mergedText += "... "
|
||||
} else {
|
||||
mergedText += " "
|
||||
}
|
||||
mergedText += next.Text
|
||||
|
||||
// Merge categories
|
||||
categories := make([]string, 0, len(current.Categories)+len(next.Categories))
|
||||
categories = append(categories, current.Categories...)
|
||||
categories = append(categories, next.Categories...)
|
||||
|
||||
return schema.Segment{
|
||||
ID: current.ID, // Will be reassigned later
|
||||
Speaker: current.Speaker,
|
||||
Start: current.Start,
|
||||
End: next.End,
|
||||
Text: mergedText,
|
||||
Categories: categories,
|
||||
}
|
||||
}
|
||||
309
internal/core/normalization/normalize_test.go
Normal file
309
internal/core/normalization/normalize_test.go
Normal file
@@ -0,0 +1,309 @@
|
||||
package normalization
|
||||
|
||||
import (
|
||||
"testing"
|
||||
|
||||
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
||||
)
|
||||
|
||||
func TestNormalizationNoOp(t *testing.T) {
|
||||
config := NormalizationConfig{
|
||||
MaxSegmentGap: 2.0,
|
||||
EllipsisGap: 1.0,
|
||||
MaxSegmentDuration: 60.0,
|
||||
MaxSegmentTokens: 100,
|
||||
}
|
||||
normalizer := NewNormalizer(config)
|
||||
|
||||
transcript := &schema.Transcript{
|
||||
Segments: []schema.Segment{
|
||||
{ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"},
|
||||
{ID: 2, Speaker: "Bob", Start: 2.0, End: 3.0, Text: "Hi"},
|
||||
},
|
||||
}
|
||||
|
||||
normalized, summary := normalizer.Normalize(transcript)
|
||||
|
||||
// Should be no-op since different speakers
|
||||
if len(normalized.Segments) != 2 {
|
||||
t.Errorf("expected 2 segments, got %d", len(normalized.Segments))
|
||||
}
|
||||
if summary.InputSegmentCount != 2 {
|
||||
t.Errorf("expected input count 2, got %d", summary.InputSegmentCount)
|
||||
}
|
||||
if summary.OutputSegmentCount != 2 {
|
||||
t.Errorf("expected output count 2, got %d", summary.OutputSegmentCount)
|
||||
}
|
||||
if summary.MergesPerformed != 0 {
|
||||
t.Errorf("expected 0 merges, got %d", summary.MergesPerformed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizationSameSpeakerMergeWithinGap(t *testing.T) {
|
||||
config := NormalizationConfig{
|
||||
MaxSegmentGap: 2.0,
|
||||
EllipsisGap: 1.0,
|
||||
MaxSegmentDuration: 60.0,
|
||||
MaxSegmentTokens: 100,
|
||||
}
|
||||
normalizer := NewNormalizer(config)
|
||||
|
||||
transcript := &schema.Transcript{
|
||||
Segments: []schema.Segment{
|
||||
{ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"},
|
||||
{ID: 2, Speaker: "Alice", Start: 1.5, End: 2.5, Text: "world"},
|
||||
},
|
||||
}
|
||||
|
||||
normalized, summary := normalizer.Normalize(transcript)
|
||||
|
||||
// Should merge since same speaker and small gap
|
||||
if len(normalized.Segments) != 1 {
|
||||
t.Errorf("expected 1 merged segment, got %d", len(normalized.Segments))
|
||||
}
|
||||
if summary.MergesPerformed != 1 {
|
||||
t.Errorf("expected 1 merge, got %d", summary.MergesPerformed)
|
||||
}
|
||||
if !strings.Contains(normalized.Segments[0].Text, "Hello") || !strings.Contains(normalized.Segments[0].Text, "world") {
|
||||
t.Errorf("expected merged text to contain both parts, got %q", normalized.Segments[0].Text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizationEllipsisInsertionAboveThreshold(t *testing.T) {
|
||||
config := NormalizationConfig{
|
||||
MaxSegmentGap: 2.0,
|
||||
EllipsisGap: 0.5,
|
||||
MaxSegmentDuration: 60.0,
|
||||
MaxSegmentTokens: 100,
|
||||
}
|
||||
normalizer := NewNormalizer(config)
|
||||
|
||||
transcript := &schema.Transcript{
|
||||
Segments: []schema.Segment{
|
||||
{ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"},
|
||||
{ID: 2, Speaker: "Alice", Start: 2.0, End: 3.0, Text: "world"},
|
||||
},
|
||||
}
|
||||
|
||||
normalized, _ := normalizer.Normalize(transcript)
|
||||
|
||||
// Should merge with ellipsis since gap (1.0) > ellipsis threshold (0.5)
|
||||
if len(normalized.Segments) != 1 {
|
||||
t.Errorf("expected 1 merged segment, got %d", len(normalized.Segments))
|
||||
}
|
||||
if !strings.Contains(normalized.Segments[0].Text, "...") {
|
||||
t.Errorf("expected ellipsis in merged text, got %q", normalized.Segments[0].Text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizationNoMergeDifferentSpeakers(t *testing.T) {
|
||||
config := NormalizationConfig{
|
||||
MaxSegmentGap: 2.0,
|
||||
EllipsisGap: 1.0,
|
||||
MaxSegmentDuration: 60.0,
|
||||
MaxSegmentTokens: 100,
|
||||
}
|
||||
normalizer := NewNormalizer(config)
|
||||
|
||||
transcript := &schema.Transcript{
|
||||
Segments: []schema.Segment{
|
||||
{ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"},
|
||||
{ID: 2, Speaker: "Bob", Start: 1.5, End: 2.5, Text: "Hi"},
|
||||
},
|
||||
}
|
||||
|
||||
normalized, summary := normalizer.Normalize(transcript)
|
||||
|
||||
// Should NOT merge since different speakers
|
||||
if len(normalized.Segments) != 2 {
|
||||
t.Errorf("expected 2 segments, got %d", len(normalized.Segments))
|
||||
}
|
||||
if summary.SkippedMerges.DifferentSpeakers != 1 {
|
||||
t.Errorf("expected 1 skipped merge for different speakers, got %d", summary.SkippedMerges.DifferentSpeakers)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizationNoMergeWhenMaxDurationExceeded(t *testing.T) {
|
||||
config := NormalizationConfig{
|
||||
MaxSegmentGap: 2.0,
|
||||
EllipsisGap: 1.0,
|
||||
MaxSegmentDuration: 1.0, // Very small max duration
|
||||
MaxSegmentTokens: 100,
|
||||
}
|
||||
normalizer := NewNormalizer(config)
|
||||
|
||||
transcript := &schema.Transcript{
|
||||
Segments: []schema.Segment{
|
||||
{ID: 1, Speaker: "Alice", Start: 0.0, End: 0.5, Text: "Hello"},
|
||||
{ID: 2, Speaker: "Alice", Start: 0.6, End: 1.5, Text: "world"},
|
||||
},
|
||||
}
|
||||
|
||||
normalized, summary := normalizer.Normalize(transcript)
|
||||
|
||||
// Should NOT merge since merged duration (1.5) > max duration (1.0)
|
||||
if len(normalized.Segments) != 2 {
|
||||
t.Errorf("expected 2 segments, got %d", len(normalized.Segments))
|
||||
}
|
||||
if summary.SkippedMerges.DurationExceeded != 1 {
|
||||
t.Errorf("expected 1 skipped merge for duration exceeded, got %d", summary.SkippedMerges.DurationExceeded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizationNoMergeWhenTokenLimitExceeded(t *testing.T) {
|
||||
config := NormalizationConfig{
|
||||
MaxSegmentGap: 2.0,
|
||||
EllipsisGap: 1.0,
|
||||
MaxSegmentDuration: 60.0,
|
||||
MaxSegmentTokens: 3, // Very small token limit
|
||||
}
|
||||
normalizer := NewNormalizer(config)
|
||||
|
||||
transcript := &schema.Transcript{
|
||||
Segments: []schema.Segment{
|
||||
{ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"},
|
||||
{ID: 2, Speaker: "Alice", Start: 1.5, End: 2.5, Text: "world"},
|
||||
},
|
||||
}
|
||||
|
||||
normalized, summary := normalizer.Normalize(transcript)
|
||||
|
||||
// Should NOT merge since merged token count would exceed limit
|
||||
if len(normalized.Segments) != 2 {
|
||||
t.Errorf("expected 2 segments, got %d", len(normalized.Segments))
|
||||
}
|
||||
if summary.SkippedMerges.TokenLimitExceeded != 1 {
|
||||
t.Errorf("expected 1 skipped merge for token limit exceeded, got %d", summary.SkippedMerges.TokenLimitExceeded)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizationChronologicalOrdering(t *testing.T) {
|
||||
config := NormalizationConfig{
|
||||
MaxSegmentGap: 2.0,
|
||||
EllipsisGap: 1.0,
|
||||
MaxSegmentDuration: 60.0,
|
||||
MaxSegmentTokens: 100,
|
||||
}
|
||||
normalizer := NewNormalizer(config)
|
||||
|
||||
transcript := &schema.Transcript{
|
||||
Segments: []schema.Segment{
|
||||
{ID: 2, Speaker: "Alice", Start: 5.0, End: 6.0, Text: "Later"},
|
||||
{ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "First"},
|
||||
{ID: 3, Speaker: "Alice", Start: 2.0, End: 3.0, Text: "Middle"},
|
||||
},
|
||||
}
|
||||
|
||||
normalized, _ := normalizer.Normalize(transcript)
|
||||
|
||||
// Should be sorted chronologically
|
||||
if len(normalized.Segments) != 1 {
|
||||
t.Errorf("expected 1 merged segment, got %d", len(normalized.Segments))
|
||||
}
|
||||
if normalized.Segments[0].Start != 0.0 {
|
||||
t.Errorf("expected first segment to start at 0.0, got %f", normalized.Segments[0].Start)
|
||||
}
|
||||
if normalized.Segments[0].End != 6.0 {
|
||||
t.Errorf("expected merged segment to end at 6.0, got %f", normalized.Segments[0].End)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizationSequentialIDReassignment(t *testing.T) {
|
||||
config := NormalizationConfig{
|
||||
MaxSegmentGap: 2.0,
|
||||
EllipsisGap: 1.0,
|
||||
MaxSegmentDuration: 60.0,
|
||||
MaxSegmentTokens: 100,
|
||||
}
|
||||
normalizer := NewNormalizer(config)
|
||||
|
||||
transcript := &schema.Transcript{
|
||||
Segments: []schema.Segment{
|
||||
{ID: 5, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"},
|
||||
{ID: 10, Speaker: "Bob", Start: 2.0, End: 3.0, Text: "Hi"},
|
||||
},
|
||||
}
|
||||
|
||||
normalized, summary := normalizer.Normalize(transcript)
|
||||
|
||||
// Should have sequential IDs starting at 1
|
||||
if len(normalized.Segments) != 2 {
|
||||
t.Errorf("expected 2 segments, got %d", len(normalized.Segments))
|
||||
}
|
||||
if normalized.Segments[0].ID != 1 {
|
||||
t.Errorf("expected first segment ID 1, got %d", normalized.Segments[0].ID)
|
||||
}
|
||||
if normalized.Segments[1].ID != 2 {
|
||||
t.Errorf("expected second segment ID 2, got %d", normalized.Segments[1].ID)
|
||||
}
|
||||
if summary.IDsReassigned != 2 {
|
||||
t.Errorf("expected 2 IDs reassigned, got %d", summary.IDsReassigned)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizationCategoryPreservation(t *testing.T) {
|
||||
config := NormalizationConfig{
|
||||
MaxSegmentGap: 2.0,
|
||||
EllipsisGap: 1.0,
|
||||
MaxSegmentDuration: 60.0,
|
||||
MaxSegmentTokens: 100,
|
||||
}
|
||||
normalizer := NewNormalizer(config)
|
||||
|
||||
transcript := &schema.Transcript{
|
||||
Segments: []schema.Segment{
|
||||
{ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello", Categories: []string{"greeting"}},
|
||||
{ID: 2, Speaker: "Alice", Start: 1.5, End: 2.5, Text: "world", Categories: []string{"response"}},
|
||||
},
|
||||
}
|
||||
|
||||
normalized, _ := normalizer.Normalize(transcript)
|
||||
|
||||
// Should merge and preserve both categories
|
||||
if len(normalized.Segments) != 1 {
|
||||
t.Errorf("expected 1 merged segment, got %d", len(normalized.Segments))
|
||||
}
|
||||
if len(normalized.Segments[0].Categories) != 2 {
|
||||
t.Errorf("expected 2 categories, got %d", len(normalized.Segments[0].Categories))
|
||||
}
|
||||
if normalized.Segments[0].Categories[0] != "greeting" || normalized.Segments[0].Categories[1] != "response" {
|
||||
t.Errorf("expected preserved categories, got %v", normalized.Segments[0].Categories)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizationSummaryCounts(t *testing.T) {
|
||||
config := NormalizationConfig{
|
||||
MaxSegmentGap: 2.0,
|
||||
EllipsisGap: 1.0,
|
||||
MaxSegmentDuration: 60.0,
|
||||
MaxSegmentTokens: 100,
|
||||
}
|
||||
normalizer := NewNormalizer(config)
|
||||
|
||||
transcript := &schema.Transcript{
|
||||
Segments: []schema.Segment{
|
||||
{ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"},
|
||||
{ID: 2, Speaker: "Alice", Start: 1.5, End: 2.5, Text: "world"},
|
||||
{ID: 3, Speaker: "Bob", Start: 3.0, End: 4.0, Text: "Hi"},
|
||||
},
|
||||
}
|
||||
|
||||
_, summary := normalizer.Normalize(transcript)
|
||||
|
||||
// Verify summary counts
|
||||
if summary.InputSegmentCount != 3 {
|
||||
t.Errorf("expected input count 3, got %d", summary.InputSegmentCount)
|
||||
}
|
||||
if summary.OutputSegmentCount != 2 {
|
||||
t.Errorf("expected output count 2, got %d", summary.OutputSegmentCount)
|
||||
}
|
||||
if summary.MergesPerformed != 1 {
|
||||
t.Errorf("expected 1 merge, got %d", summary.MergesPerformed)
|
||||
}
|
||||
if summary.IDsReassigned != 3 {
|
||||
t.Errorf("expected 3 IDs reassigned, got %d", summary.IDsReassigned)
|
||||
}
|
||||
if summary.SkippedMerges.DifferentSpeakers != 1 {
|
||||
t.Errorf("expected 1 skipped merge for different speakers, got %d", summary.SkippedMerges.DifferentSpeakers)
|
||||
}
|
||||
}
|
||||
33
internal/core/normalization/tokens.go
Normal file
33
internal/core/normalization/tokens.go
Normal file
@@ -0,0 +1,33 @@
|
||||
package normalization
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"unicode"
|
||||
)
|
||||
|
||||
// SimpleTokenEstimator provides a basic deterministic token estimation.
|
||||
// This is a placeholder that can be replaced with a more sophisticated estimator later.
|
||||
type SimpleTokenEstimator struct{}
|
||||
|
||||
// EstimateTokens provides a rough estimate of the number of tokens in the given text.
|
||||
// This implementation uses a simple heuristic: count words and punctuation as tokens.
|
||||
func (e *SimpleTokenEstimator) EstimateTokens(text string) int {
|
||||
if text == "" {
|
||||
return 0
|
||||
}
|
||||
|
||||
// Simple heuristic: split on whitespace and count non-empty segments
|
||||
words := strings.Fields(text)
|
||||
tokenCount := len(words)
|
||||
|
||||
// Add some estimate for punctuation that might be separate tokens
|
||||
punctuationCount := 0
|
||||
for _, r := range text {
|
||||
if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
|
||||
punctuationCount++
|
||||
}
|
||||
}
|
||||
|
||||
// Rough estimate: each word is a token, plus half the punctuation as separate tokens
|
||||
return tokenCount + (punctuationCount / 2)
|
||||
}
|
||||
Reference in New Issue
Block a user