Add token-bounded transcript chunking

internal/core/chunking/tokens.go

@@ -0,0 +1,55 @@
+package chunking
+
+import (
+	"strings"
+	"unicode"
+)
+
+// TokenEstimator provides a deterministic token estimation suitable for prompt budgeting.
+// The estimator is approximate but stable, isolated, and replaceable.
+type TokenEstimator interface {
+	EstimateTokens(text string) int
+}
+
+// SimpleTokenEstimator provides a basic deterministic token estimation.
+// This uses a simple heuristic based on word count and punctuation.
+type SimpleTokenEstimator struct{}
+
+// NewSimpleTokenEstimator creates a new simple token estimator.
+func NewSimpleTokenEstimator() *SimpleTokenEstimator {
+	return &SimpleTokenEstimator{}
+}
+
+// EstimateTokens provides a rough estimate of the number of tokens in the given text.
+// This implementation uses a simple heuristic: count words and punctuation as tokens.
+// The estimate is deterministic and stable for the same input text.
+func (e *SimpleTokenEstimator) EstimateTokens(text string) int {
+	if text == "" {
+		return 0
+	}
+
+	// Simple heuristic: split on whitespace and count non-empty segments
+	words := strings.Fields(text)
+	tokenCount := len(words)
+
+	// Add some estimate for punctuation that might be separate tokens
+	punctuationCount := 0
+	for _, r := range text {
+		if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
+			punctuationCount++
+		}
+	}
+
+	// Rough estimate: each word is a token, plus half the punctuation as separate tokens
+	return tokenCount + (punctuationCount / 2)
+}
+
+// ConstTokenEstimator returns a constant token count for testing purposes.
+type ConstTokenEstimator struct {
+	Tokens int
+}
+
+// EstimateTokens returns the configured constant token count.
+func (e *ConstTokenEstimator) EstimateTokens(text string) int {
+	return e.Tokens
+}