60 lines
1.7 KiB
Go
60 lines
1.7 KiB
Go
package chunking
|
|
|
|
import (
|
|
"testing"
|
|
)
|
|
|
|
func TestSimpleTokenEstimator(t *testing.T) {
|
|
estimator := NewSimpleTokenEstimator()
|
|
|
|
tests := []struct {
|
|
name string
|
|
text string
|
|
expected int
|
|
}{
|
|
{"empty string", "", 0},
|
|
{"single word", "hello", 1},
|
|
{"two words", "hello world", 2},
|
|
{"with punctuation", "hello, world!", 3}, // 2 words + 2 punctuation/2 = 3
|
|
{"multiple sentences", "Hello world. This is a test.", 7}, // 7 words + 2 punctuation/2 = 8? Actually "Hello world." has 3 punctuation
|
|
{"with apostrophes", "don't won't can't", 3},
|
|
{"with hyphens", "well-known state-of-the-art", 2}, // hyphens don't count
|
|
{"unicode text", "café naïve", 2},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
got := estimator.EstimateTokens(tt.text)
|
|
if got != tt.expected {
|
|
t.Errorf("EstimateTokens(%q) = %d, want %d", tt.text, got, tt.expected)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestConstTokenEstimator(t *testing.T) {
|
|
estimator := &ConstTokenEstimator{Tokens: 42}
|
|
|
|
if got := estimator.EstimateTokens("any text"); got != 42 {
|
|
t.Errorf("ConstTokenEstimator.EstimateTokens = %d, want 42", got)
|
|
}
|
|
|
|
if got := estimator.EstimateTokens(""); got != 42 {
|
|
t.Errorf("ConstTokenEstimator.EstimateTokens(empty) = %d, want 42", got)
|
|
}
|
|
}
|
|
|
|
func TestTokenEstimatorDeterminism(t *testing.T) {
|
|
estimator := NewSimpleTokenEstimator()
|
|
text := "The quick brown fox jumps over the lazy dog. Hello, world!"
|
|
|
|
// Run multiple times and verify same result
|
|
first := estimator.EstimateTokens(text)
|
|
for i := 0; i < 10; i++ {
|
|
got := estimator.EstimateTokens(text)
|
|
if got != first {
|
|
t.Errorf("EstimateTokens not deterministic: iteration %d got %d, first was %d", i, got, first)
|
|
}
|
|
}
|
|
}
|