eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Add deterministic transcript normalization

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-11 00:12:19 +00:00
parent 3cfa4b6e8a
commit 0b1b670baf
3 changed files with 522 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

180
internal/core/normalization/normalize.go Normal file
View File

@@ -0,0 +1,180 @@
package normalization
import (
"sort"
"strings"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
)
// NormalizationConfig holds configuration parameters for transcript normalization
type NormalizationConfig struct {
MaxSegmentGap float64 // Maximum gap between segments to consider for merging (seconds)
EllipsisGap float64 // Gap threshold above which to insert ellipsis (seconds)
MaxSegmentDuration float64 // Maximum duration for a merged segment (seconds)
MaxSegmentTokens int // Maximum token estimate for a merged segment
}
// NormalizationSummary records the results and statistics of normalization
type NormalizationSummary struct {
InputSegmentCount int `json:"input_segment_count"`
OutputSegmentCount int `json:"output_segment_count"`
MergesPerformed int `json:"merges_performed"`
IDsReassigned int `json:"ids_reassigned"`
SkippedMerges struct {
DifferentSpeakers int `json:"different_speakers"`
GapTooLarge int `json:"gap_too_large"`
DurationExceeded int `json:"duration_exceeded"`
TokenLimitExceeded int `json:"token_limit_exceeded"`
} `json:"skipped_merges"`
}
// NormalizeTranscript performs deterministic normalization on a transcript
type NormalizeTranscript struct {
config NormalizationConfig
estimator *SimpleTokenEstimator
}
// NewNormalizer creates a new transcript normalizer with the given configuration
func NewNormalizer(config NormalizationConfig) *NormalizeTranscript {
return &NormalizeTranscript{
config: config,
estimator: &SimpleTokenEstimator{},
}
}
// Normalize performs deterministic normalization on the given transcript
func (n *NormalizeTranscript) Normalize(transcript *schema.Transcript) (*schema.Transcript, *NormalizationSummary) {
summary := &NormalizationSummary{
InputSegmentCount: len(transcript.Segments),
}
if len(transcript.Segments) == 0 {
return transcript, summary
}
// Sort segments chronologically
sortedSegments := make([]schema.Segment, len(transcript.Segments))
copy(sortedSegments, transcript.Segments)
sort.Slice(sortedSegments, func(i, j int) bool {
return sortedSegments[i].Start < sortedSegments[j].Start
})
// Merge same-speaker adjacent segments
var normalizedSegments []schema.Segment
var currentSegment schema.Segment
var sourceIDs []int
for i, segment := range sortedSegments {
if i == 0 {
// Initialize with first segment
currentSegment = segment
sourceIDs = []int{segment.ID}
continue
}
// Check if we should merge with current segment
gap := segment.Start - currentSegment.End
merge, reason := shouldMergeWithReason(n.config, n.estimator, &currentSegment, &segment, gap)
if merge {
summary.MergesPerformed++
currentSegment = mergeSegments(&currentSegment, &segment, gap, n.config.EllipsisGap)
sourceIDs = append(sourceIDs, segment.ID)
} else {
// Track the reason for not merging
switch reason {
case "different_speakers":
summary.SkippedMerges.DifferentSpeakers++
case "gap_too_large":
summary.SkippedMerges.GapTooLarge++
case "duration_exceeded":
summary.SkippedMerges.DurationExceeded++
case "token_limit_exceeded":
summary.SkippedMerges.TokenLimitExceeded++
}
// Finalize current segment and start new one
normalizedSegments = append(normalizedSegments, currentSegment)
currentSegment = segment
sourceIDs = []int{segment.ID}
}
}
// Add the last segment
if len(currentSegment.Text) > 0 {
normalizedSegments = append(normalizedSegments, currentSegment)
}
// Reassign sequential IDs starting at 1
for i, segment := range normalizedSegments {
if segment.ID != i+1 {
summary.IDsReassigned++
}
normalizedSegments[i].ID = i + 1
}
summary.OutputSegmentCount = len(normalizedSegments)
return &schema.Transcript{Segments: normalizedSegments}, summary
}
func shouldMerge(config NormalizationConfig, estimator *SimpleTokenEstimator,
current, next *schema.Segment, gap float64) bool {
// Don't merge if different speakers
if current.Speaker != next.Speaker {
return false
}
// Don't merge if gap is too large
if gap > config.MaxSegmentGap {
return false
}
// Calculate what the merged segment would look like
mergedText := current.Text
if gap >= config.EllipsisGap {
mergedText += "... "
} else {
mergedText += " "
}
mergedText += next.Text
mergedDuration := next.End - current.Start
// Don't merge if duration would be exceeded
if mergedDuration > config.MaxSegmentDuration {
return false
}
// Don't merge if token estimate would be exceeded
tokenEstimate := estimator.EstimateTokens(mergedText)
if tokenEstimate > config.MaxSegmentTokens {
return false
}
return true
}
func mergeSegments(current, next *schema.Segment, gap, ellipsisGap float64) schema.Segment {
mergedText := current.Text
if gap >= ellipsisGap {
mergedText += "... "
} else {
mergedText += " "
}
mergedText += next.Text
// Merge categories
categories := make([]string, 0, len(current.Categories)+len(next.Categories))
categories = append(categories, current.Categories...)
categories = append(categories, next.Categories...)
return schema.Segment{
ID: current.ID, // Will be reassigned later
Speaker: current.Speaker,
Start: current.Start,
End: next.End,
Text: mergedText,
Categories: categories,
}
}

309
internal/core/normalization/normalize_test.go Normal file
View File

@@ -0,0 +1,309 @@
package normalization
import (
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
)
func TestNormalizationNoOp(t *testing.T) {
config := NormalizationConfig{
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
}
normalizer := NewNormalizer(config)
transcript := &schema.Transcript{
Segments: []schema.Segment{
{ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"},
{ID: 2, Speaker: "Bob", Start: 2.0, End: 3.0, Text: "Hi"},
},
}
normalized, summary := normalizer.Normalize(transcript)
// Should be no-op since different speakers
if len(normalized.Segments) != 2 {
t.Errorf("expected 2 segments, got %d", len(normalized.Segments))
}
if summary.InputSegmentCount != 2 {
t.Errorf("expected input count 2, got %d", summary.InputSegmentCount)
}
if summary.OutputSegmentCount != 2 {
t.Errorf("expected output count 2, got %d", summary.OutputSegmentCount)
}
if summary.MergesPerformed != 0 {
t.Errorf("expected 0 merges, got %d", summary.MergesPerformed)
}
}
func TestNormalizationSameSpeakerMergeWithinGap(t *testing.T) {
config := NormalizationConfig{
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
}
normalizer := NewNormalizer(config)
transcript := &schema.Transcript{
Segments: []schema.Segment{
{ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"},
{ID: 2, Speaker: "Alice", Start: 1.5, End: 2.5, Text: "world"},
},
}
normalized, summary := normalizer.Normalize(transcript)
// Should merge since same speaker and small gap
if len(normalized.Segments) != 1 {
t.Errorf("expected 1 merged segment, got %d", len(normalized.Segments))
}
if summary.MergesPerformed != 1 {
t.Errorf("expected 1 merge, got %d", summary.MergesPerformed)
}
if !strings.Contains(normalized.Segments[0].Text, "Hello") || !strings.Contains(normalized.Segments[0].Text, "world") {
t.Errorf("expected merged text to contain both parts, got %q", normalized.Segments[0].Text)
}
}
func TestNormalizationEllipsisInsertionAboveThreshold(t *testing.T) {
config := NormalizationConfig{
MaxSegmentGap: 2.0,
EllipsisGap: 0.5,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
}
normalizer := NewNormalizer(config)
transcript := &schema.Transcript{
Segments: []schema.Segment{
{ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"},
{ID: 2, Speaker: "Alice", Start: 2.0, End: 3.0, Text: "world"},
},
}
normalized, _ := normalizer.Normalize(transcript)
// Should merge with ellipsis since gap (1.0) > ellipsis threshold (0.5)
if len(normalized.Segments) != 1 {
t.Errorf("expected 1 merged segment, got %d", len(normalized.Segments))
}
if !strings.Contains(normalized.Segments[0].Text, "...") {
t.Errorf("expected ellipsis in merged text, got %q", normalized.Segments[0].Text)
}
}
func TestNormalizationNoMergeDifferentSpeakers(t *testing.T) {
config := NormalizationConfig{
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
}
normalizer := NewNormalizer(config)
transcript := &schema.Transcript{
Segments: []schema.Segment{
{ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"},
{ID: 2, Speaker: "Bob", Start: 1.5, End: 2.5, Text: "Hi"},
},
}
normalized, summary := normalizer.Normalize(transcript)
// Should NOT merge since different speakers
if len(normalized.Segments) != 2 {
t.Errorf("expected 2 segments, got %d", len(normalized.Segments))
}
if summary.SkippedMerges.DifferentSpeakers != 1 {
t.Errorf("expected 1 skipped merge for different speakers, got %d", summary.SkippedMerges.DifferentSpeakers)
}
}
func TestNormalizationNoMergeWhenMaxDurationExceeded(t *testing.T) {
config := NormalizationConfig{
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 1.0, // Very small max duration
MaxSegmentTokens: 100,
}
normalizer := NewNormalizer(config)
transcript := &schema.Transcript{
Segments: []schema.Segment{
{ID: 1, Speaker: "Alice", Start: 0.0, End: 0.5, Text: "Hello"},
{ID: 2, Speaker: "Alice", Start: 0.6, End: 1.5, Text: "world"},
},
}
normalized, summary := normalizer.Normalize(transcript)
// Should NOT merge since merged duration (1.5) > max duration (1.0)
if len(normalized.Segments) != 2 {
t.Errorf("expected 2 segments, got %d", len(normalized.Segments))
}
if summary.SkippedMerges.DurationExceeded != 1 {
t.Errorf("expected 1 skipped merge for duration exceeded, got %d", summary.SkippedMerges.DurationExceeded)
}
}
func TestNormalizationNoMergeWhenTokenLimitExceeded(t *testing.T) {
config := NormalizationConfig{
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 3, // Very small token limit
}
normalizer := NewNormalizer(config)
transcript := &schema.Transcript{
Segments: []schema.Segment{
{ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"},
{ID: 2, Speaker: "Alice", Start: 1.5, End: 2.5, Text: "world"},
},
}
normalized, summary := normalizer.Normalize(transcript)
// Should NOT merge since merged token count would exceed limit
if len(normalized.Segments) != 2 {
t.Errorf("expected 2 segments, got %d", len(normalized.Segments))
}
if summary.SkippedMerges.TokenLimitExceeded != 1 {
t.Errorf("expected 1 skipped merge for token limit exceeded, got %d", summary.SkippedMerges.TokenLimitExceeded)
}
}
func TestNormalizationChronologicalOrdering(t *testing.T) {
config := NormalizationConfig{
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
}
normalizer := NewNormalizer(config)
transcript := &schema.Transcript{
Segments: []schema.Segment{
{ID: 2, Speaker: "Alice", Start: 5.0, End: 6.0, Text: "Later"},
{ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "First"},
{ID: 3, Speaker: "Alice", Start: 2.0, End: 3.0, Text: "Middle"},
},
}
normalized, _ := normalizer.Normalize(transcript)
// Should be sorted chronologically
if len(normalized.Segments) != 1 {
t.Errorf("expected 1 merged segment, got %d", len(normalized.Segments))
}
if normalized.Segments[0].Start != 0.0 {
t.Errorf("expected first segment to start at 0.0, got %f", normalized.Segments[0].Start)
}
if normalized.Segments[0].End != 6.0 {
t.Errorf("expected merged segment to end at 6.0, got %f", normalized.Segments[0].End)
}
}
func TestNormalizationSequentialIDReassignment(t *testing.T) {
config := NormalizationConfig{
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
}
normalizer := NewNormalizer(config)
transcript := &schema.Transcript{
Segments: []schema.Segment{
{ID: 5, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"},
{ID: 10, Speaker: "Bob", Start: 2.0, End: 3.0, Text: "Hi"},
},
}
normalized, summary := normalizer.Normalize(transcript)
// Should have sequential IDs starting at 1
if len(normalized.Segments) != 2 {
t.Errorf("expected 2 segments, got %d", len(normalized.Segments))
}
if normalized.Segments[0].ID != 1 {
t.Errorf("expected first segment ID 1, got %d", normalized.Segments[0].ID)
}
if normalized.Segments[1].ID != 2 {
t.Errorf("expected second segment ID 2, got %d", normalized.Segments[1].ID)
}
if summary.IDsReassigned != 2 {
t.Errorf("expected 2 IDs reassigned, got %d", summary.IDsReassigned)
}
}
func TestNormalizationCategoryPreservation(t *testing.T) {
config := NormalizationConfig{
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
}
normalizer := NewNormalizer(config)
transcript := &schema.Transcript{
Segments: []schema.Segment{
{ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello", Categories: []string{"greeting"}},
{ID: 2, Speaker: "Alice", Start: 1.5, End: 2.5, Text: "world", Categories: []string{"response"}},
},
}
normalized, _ := normalizer.Normalize(transcript)
// Should merge and preserve both categories
if len(normalized.Segments) != 1 {
t.Errorf("expected 1 merged segment, got %d", len(normalized.Segments))
}
if len(normalized.Segments[0].Categories) != 2 {
t.Errorf("expected 2 categories, got %d", len(normalized.Segments[0].Categories))
}
if normalized.Segments[0].Categories[0] != "greeting" || normalized.Segments[0].Categories[1] != "response" {
t.Errorf("expected preserved categories, got %v", normalized.Segments[0].Categories)
}
}
func TestNormalizationSummaryCounts(t *testing.T) {
config := NormalizationConfig{
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
}
normalizer := NewNormalizer(config)
transcript := &schema.Transcript{
Segments: []schema.Segment{
{ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"},
{ID: 2, Speaker: "Alice", Start: 1.5, End: 2.5, Text: "world"},
{ID: 3, Speaker: "Bob", Start: 3.0, End: 4.0, Text: "Hi"},
},
}
_, summary := normalizer.Normalize(transcript)
// Verify summary counts
if summary.InputSegmentCount != 3 {
t.Errorf("expected input count 3, got %d", summary.InputSegmentCount)
}
if summary.OutputSegmentCount != 2 {
t.Errorf("expected output count 2, got %d", summary.OutputSegmentCount)
}
if summary.MergesPerformed != 1 {
t.Errorf("expected 1 merge, got %d", summary.MergesPerformed)
}
if summary.IDsReassigned != 3 {
t.Errorf("expected 3 IDs reassigned, got %d", summary.IDsReassigned)
}
if summary.SkippedMerges.DifferentSpeakers != 1 {
t.Errorf("expected 1 skipped merge for different speakers, got %d", summary.SkippedMerges.DifferentSpeakers)
}
}

33
internal/core/normalization/tokens.go Normal file
View File

@@ -0,0 +1,33 @@
package normalization
import (
"strings"
"unicode"
)
// SimpleTokenEstimator provides a basic deterministic token estimation.
// This is a placeholder that can be replaced with a more sophisticated estimator later.
type SimpleTokenEstimator struct{}
// EstimateTokens provides a rough estimate of the number of tokens in the given text.
// This implementation uses a simple heuristic: count words and punctuation as tokens.
func (e *SimpleTokenEstimator) EstimateTokens(text string) int {
if text == "" {
return 0
}
// Simple heuristic: split on whitespace and count non-empty segments
words := strings.Fields(text)
tokenCount := len(words)
// Add some estimate for punctuation that might be separate tokens
punctuationCount := 0
for _, r := range text {
if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
punctuationCount++
}
}
// Rough estimate: each word is a token, plus half the punctuation as separate tokens
return tokenCount + (punctuationCount / 2)
}