Add deterministic transcript normalization

internal/core/normalization/tokens.go

@@ -0,0 +1,33 @@
+package normalization
+
+import (
+	"strings"
+	"unicode"
+)
+
+// SimpleTokenEstimator provides a basic deterministic token estimation.
+// This is a placeholder that can be replaced with a more sophisticated estimator later.
+type SimpleTokenEstimator struct{}
+
+// EstimateTokens provides a rough estimate of the number of tokens in the given text.
+// This implementation uses a simple heuristic: count words and punctuation as tokens.
+func (e *SimpleTokenEstimator) EstimateTokens(text string) int {
+	if text == "" {
+		return 0
+	}
+
+	// Simple heuristic: split on whitespace and count non-empty segments
+	words := strings.Fields(text)
+	tokenCount := len(words)
+
+	// Add some estimate for punctuation that might be separate tokens
+	punctuationCount := 0
+	for _, r := range text {
+		if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
+			punctuationCount++
+		}
+	}
+
+	// Rough estimate: each word is a token, plus half the punctuation as separate tokens
+	return tokenCount + (punctuationCount / 2)
+}