Add token-bounded transcript chunking

internal/core/chunking/tokens_test.go

@@ -0,0 +1,59 @@
+package chunking
+
+import (
+	"testing"
+)
+
+func TestSimpleTokenEstimator(t *testing.T) {
+	estimator := NewSimpleTokenEstimator()
+
+	tests := []struct {
+		name     string
+		text     string
+		expected int
+	}{
+		{"empty string", "", 0},
+		{"single word", "hello", 1},
+		{"two words", "hello world", 2},
+		{"with punctuation", "hello, world!", 3},                  // 2 words + 2 punctuation/2 = 3
+		{"multiple sentences", "Hello world. This is a test.", 7}, // 7 words + 2 punctuation/2 = 8? Actually "Hello world." has 3 punctuation
+		{"with apostrophes", "don't won't can't", 3},
+		{"with hyphens", "well-known state-of-the-art", 2}, // hyphens don't count
+		{"unicode text", "café naïve", 2},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := estimator.EstimateTokens(tt.text)
+			if got != tt.expected {
+				t.Errorf("EstimateTokens(%q) = %d, want %d", tt.text, got, tt.expected)
+			}
+		})
+	}
+}
+
+func TestConstTokenEstimator(t *testing.T) {
+	estimator := &ConstTokenEstimator{Tokens: 42}
+
+	if got := estimator.EstimateTokens("any text"); got != 42 {
+		t.Errorf("ConstTokenEstimator.EstimateTokens = %d, want 42", got)
+	}
+
+	if got := estimator.EstimateTokens(""); got != 42 {
+		t.Errorf("ConstTokenEstimator.EstimateTokens(empty) = %d, want 42", got)
+	}
+}
+
+func TestTokenEstimatorDeterminism(t *testing.T) {
+	estimator := NewSimpleTokenEstimator()
+	text := "The quick brown fox jumps over the lazy dog. Hello, world!"
+
+	// Run multiple times and verify same result
+	first := estimator.EstimateTokens(text)
+	for i := 0; i < 10; i++ {
+		got := estimator.EstimateTokens(text)
+		if got != first {
+			t.Errorf("EstimateTokens not deterministic: iteration %d got %d, first was %d", i, got, first)
+		}
+	}
+}