Complete Phase 2 Go normalization foundation

internal/core/normalization/normalize.go

@@ -2,7 +2,6 @@ package normalization
 
 import (
 	"sort"
-	"strings"
 
 	"gitea.maximumdirect.net/eric/audita/internal/core/schema"
 )
@@ -63,13 +62,11 @@ func (n *NormalizeTranscript) Normalize(transcript *schema.Transcript) (*schema.
 	// Merge same-speaker adjacent segments
 	var normalizedSegments []schema.Segment
 	var currentSegment schema.Segment
-	var sourceIDs []int
 
 	for i, segment := range sortedSegments {
 		if i == 0 {
 			// Initialize with first segment
 			currentSegment = segment
-			sourceIDs = []int{segment.ID}
 			continue
 		}
 
@@ -79,7 +76,6 @@ func (n *NormalizeTranscript) Normalize(transcript *schema.Transcript) (*schema.
 		if merge {
 			summary.MergesPerformed++
 			currentSegment = mergeSegments(&currentSegment, &segment, gap, n.config.EllipsisGap)
-			sourceIDs = append(sourceIDs, segment.ID)
 		} else {
 			// Track the reason for not merging
 			switch reason {
@@ -95,7 +91,6 @@ func (n *NormalizeTranscript) Normalize(transcript *schema.Transcript) (*schema.
 			// Finalize current segment and start new one
 			normalizedSegments = append(normalizedSegments, currentSegment)
 			currentSegment = segment
-			sourceIDs = []int{segment.ID}
 		}
 	}
 
@@ -155,6 +150,37 @@ func shouldMerge(config NormalizationConfig, estimator *SimpleTokenEstimator,
 	return true
 }
 
+func shouldMergeWithReason(config NormalizationConfig, estimator *SimpleTokenEstimator,
+	current, next *schema.Segment, gap float64) (bool, string) {
+
+	if current.Speaker != next.Speaker {
+		return false, "different_speakers"
+	}
+	if gap > config.MaxSegmentGap {
+		return false, "gap_too_large"
+	}
+
+	mergedText := current.Text
+	if gap >= config.EllipsisGap {
+		mergedText += "... "
+	} else {
+		mergedText += " "
+	}
+	mergedText += next.Text
+
+	mergedDuration := next.End - current.Start
+	if mergedDuration > config.MaxSegmentDuration {
+		return false, "duration_exceeded"
+	}
+
+	tokenEstimate := estimator.EstimateTokens(mergedText)
+	if tokenEstimate > config.MaxSegmentTokens {
+		return false, "token_limit_exceeded"
+	}
+
+	return true, ""
+}
+
 func mergeSegments(current, next *schema.Segment, gap, ellipsisGap float64) schema.Segment {
 	mergedText := current.Text
 	if gap >= ellipsisGap {