eric/audita
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Complete Phase 2 Go normalization foundation

Browse Source
This commit is contained in:
Eric Rakestraw
2026-05-11 00:41:29 +00:00
parent d847168ecd
commit 10377876e4
11 changed files with 480 additions and 1241 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
internal/core/normalization/normalize.go
View File

@@ -2,7 +2,6 @@ package normalization
import (
"sort"
"strings"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
)
@@ -63,13 +62,11 @@ func (n *NormalizeTranscript) Normalize(transcript *schema.Transcript) (*schema.
// Merge same-speaker adjacent segments
var normalizedSegments []schema.Segment
var currentSegment schema.Segment
var sourceIDs []int
for i, segment := range sortedSegments {
if i == 0 {
// Initialize with first segment
currentSegment = segment
sourceIDs = []int{segment.ID}
continue
}
@@ -79,7 +76,6 @@ func (n *NormalizeTranscript) Normalize(transcript *schema.Transcript) (*schema.
if merge {
summary.MergesPerformed++
currentSegment = mergeSegments(&currentSegment, &segment, gap, n.config.EllipsisGap)
sourceIDs = append(sourceIDs, segment.ID)
} else {
// Track the reason for not merging
switch reason {
@@ -95,7 +91,6 @@ func (n *NormalizeTranscript) Normalize(transcript *schema.Transcript) (*schema.
// Finalize current segment and start new one
normalizedSegments = append(normalizedSegments, currentSegment)
currentSegment = segment
sourceIDs = []int{segment.ID}
}
}
@@ -155,6 +150,37 @@ func shouldMerge(config NormalizationConfig, estimator *SimpleTokenEstimator,
return true
}
func shouldMergeWithReason(config NormalizationConfig, estimator *SimpleTokenEstimator,
current, next *schema.Segment, gap float64) (bool, string) {
if current.Speaker != next.Speaker {
return false, "different_speakers"
}
if gap > config.MaxSegmentGap {
return false, "gap_too_large"
}
mergedText := current.Text
if gap >= config.EllipsisGap {
mergedText += "... "
} else {
mergedText += " "
}
mergedText += next.Text
mergedDuration := next.End - current.Start
if mergedDuration > config.MaxSegmentDuration {
return false, "duration_exceeded"
}
tokenEstimate := estimator.EstimateTokens(mergedText)
if tokenEstimate > config.MaxSegmentTokens {
return false, "token_limit_exceeded"
}
return true, ""
}
func mergeSegments(current, next *schema.Segment, gap, ellipsisGap float64) schema.Segment {
mergedText := current.Text
if gap >= ellipsisGap {

131
internal/core/normalization/normalize_test.go
View File

@@ -1,6 +1,8 @@
package normalization
import (
"os"
"strings"
"testing"
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
@@ -155,7 +157,7 @@ func TestNormalizationNoMergeWhenTokenLimitExceeded(t *testing.T) {
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 3, // Very small token limit
MaxSegmentTokens: 1, // Very small token limit
}
normalizer := NewNormalizer(config)
@@ -273,10 +275,10 @@ func TestNormalizationCategoryPreservation(t *testing.T) {
func TestNormalizationGoldenBareArrayTranscript(t *testing.T) {
config := NormalizationConfig{
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
}
normalizer := NewNormalizer(config)
@@ -316,20 +318,20 @@ func TestNormalizationGoldenBareArrayTranscript(t *testing.T) {
t.Fatalf("failed to read golden fixture: %v", err)
}
var expectedTranscript schema.Transcript
if err := schema.ParseTranscriptJSON(expectedRaw); err != nil {
expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
if err != nil {
t.Fatalf("failed to parse golden transcript: %v", err)
}
assertTranscriptsEqual(t, normalized, &expectedTranscript)
assertTranscriptsEqual(t, normalized, expectedTranscript)
}
func TestNormalizationGoldenObjectWithSegmentsTranscript(t *testing.T) {
config := NormalizationConfig{
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
}
normalizer := NewNormalizer(config)
@@ -369,20 +371,20 @@ func TestNormalizationGoldenObjectWithSegmentsTranscript(t *testing.T) {
t.Fatalf("failed to read golden fixture: %v", err)
}
var expectedTranscript schema.Transcript
if err := schema.ParseTranscriptJSON(expectedRaw); err != nil {
expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
if err != nil {
t.Fatalf("failed to parse golden transcript: %v", err)
}
assertTranscriptsEqual(t, normalized, &expectedTranscript)
assertTranscriptsEqual(t, normalized, expectedTranscript)
}
func TestNormalizationGoldenTranscriptWithCategories(t *testing.T) {
config := NormalizationConfig{
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
}
normalizer := NewNormalizer(config)
@@ -422,20 +424,20 @@ func TestNormalizationGoldenTranscriptWithCategories(t *testing.T) {
t.Fatalf("failed to read golden fixture: %v", err)
}
var expectedTranscript schema.Transcript
if err := schema.ParseTranscriptJSON(expectedRaw); err != nil {
expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
if err != nil {
t.Fatalf("failed to parse golden transcript: %v", err)
}
assertTranscriptsEqual(t, normalized, &expectedTranscript)
assertTranscriptsEqual(t, normalized, expectedTranscript)
}
func TestNormalizationGoldenTranscriptWithOriginalIDs(t *testing.T) {
config := NormalizationConfig{
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
}
normalizer := NewNormalizer(config)
@@ -475,20 +477,20 @@ func TestNormalizationGoldenTranscriptWithOriginalIDs(t *testing.T) {
t.Fatalf("failed to read golden fixture: %v", err)
}
var expectedTranscript schema.Transcript
if err := schema.ParseTranscriptJSON(expectedRaw); err != nil {
expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
if err != nil {
t.Fatalf("failed to parse golden transcript: %v", err)
}
assertTranscriptsEqual(t, normalized, &expectedTranscript)
assertTranscriptsEqual(t, normalized, expectedTranscript)
}
func TestNormalizationGoldenSameSpeakerMerge(t *testing.T) {
config := NormalizationConfig{
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
}
normalizer := NewNormalizer(config)
@@ -528,20 +530,20 @@ func TestNormalizationGoldenSameSpeakerMerge(t *testing.T) {
t.Fatalf("failed to read golden fixture: %v", err)
}
var expectedTranscript schema.Transcript
if err := schema.ParseTranscriptJSON(expectedRaw); err != nil {
expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
if err != nil {
t.Fatalf("failed to parse golden transcript: %v", err)
}
assertTranscriptsEqual(t, normalized, &expectedTranscript)
assertTranscriptsEqual(t, normalized, expectedTranscript)
}
func TestNormalizationGoldenEllipsisMerge(t *testing.T) {
config := NormalizationConfig{
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
}
normalizer := NewNormalizer(config)
@@ -581,20 +583,20 @@ func TestNormalizationGoldenEllipsisMerge(t *testing.T) {
t.Fatalf("failed to read golden fixture: %v", err)
}
var expectedTranscript schema.Transcript
if err := schema.ParseTranscriptJSON(expectedRaw); err != nil {
expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
if err != nil {
t.Fatalf("failed to parse golden transcript: %v", err)
}
assertTranscriptsEqual(t, normalized, &expectedTranscript)
assertTranscriptsEqual(t, normalized, expectedTranscript)
}
func TestNormalizationGoldenDifferentSpeakersNoMerge(t *testing.T) {
config := NormalizationConfig{
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
MaxSegmentGap: 2.0,
EllipsisGap: 1.0,
MaxSegmentDuration: 60.0,
MaxSegmentTokens: 100,
}
normalizer := NewNormalizer(config)
@@ -634,12 +636,12 @@ func TestNormalizationGoldenDifferentSpeakersNoMerge(t *testing.T) {
t.Fatalf("failed to read golden fixture: %v", err)
}
var expectedTranscript schema.Transcript
if err := schema.ParseTranscriptJSON(expectedRaw); err != nil {
expectedTranscript, err := schema.ParseTranscriptJSON(expectedRaw)
if err != nil {
t.Fatalf("failed to parse golden transcript: %v", err)
}
assertTranscriptsEqual(t, normalized, &expectedTranscript)
assertTranscriptsEqual(t, normalized, expectedTranscript)
}
func assertTranscriptsEqual(t *testing.T, actual, expected *schema.Transcript) {
@@ -679,33 +681,4 @@ func assertTranscriptsEqual(t *testing.T, actual, expected *schema.Transcript) {
}
}
}
}
normalizer := NewNormalizer(config)
transcript := &schema.Transcript{
Segments: []schema.Segment{
{ID: 1, Speaker: "Alice", Start: 0.0, End: 1.0, Text: "Hello"},
{ID: 2, Speaker: "Alice", Start: 1.5, End: 2.5, Text: "world"},
{ID: 3, Speaker: "Bob", Start: 3.0, End: 4.0, Text: "Hi"},
},
}
_, summary := normalizer.Normalize(transcript)
// Verify summary counts
if summary.InputSegmentCount != 3 {
t.Errorf("expected input count 3, got %d", summary.InputSegmentCount)
}
if summary.OutputSegmentCount != 2 {
t.Errorf("expected output count 2, got %d", summary.OutputSegmentCount)
}
if summary.MergesPerformed != 1 {
t.Errorf("expected 1 merge, got %d", summary.MergesPerformed)
}
if summary.IDsReassigned != 3 {
t.Errorf("expected 3 IDs reassigned, got %d", summary.IDsReassigned)
}
if summary.SkippedMerges.DifferentSpeakers != 1 {
t.Errorf("expected 1 skipped merge for different speakers, got %d", summary.SkippedMerges.DifferentSpeakers)
}
}

4
internal/core/normalization/testdata/transcript_same_speaker_merge.golden.json vendored
View File

@@ -3,7 +3,7 @@
"id": 1,
"speaker": "Alice",
"start": 0.0,
"end": 2.5,
"end": 2.2,
"text": "Hello world"
}
]
]