34 lines
998 B
Go
34 lines
998 B
Go
package normalization
|
|
|
|
import (
|
|
"strings"
|
|
"unicode"
|
|
)
|
|
|
|
// SimpleTokenEstimator provides a basic deterministic token estimation.
|
|
// This is a placeholder that can be replaced with a more sophisticated estimator later.
|
|
type SimpleTokenEstimator struct{}
|
|
|
|
// EstimateTokens provides a rough estimate of the number of tokens in the given text.
|
|
// This implementation uses a simple heuristic: count words and punctuation as tokens.
|
|
func (e *SimpleTokenEstimator) EstimateTokens(text string) int {
|
|
if text == "" {
|
|
return 0
|
|
}
|
|
|
|
// Simple heuristic: split on whitespace and count non-empty segments
|
|
words := strings.Fields(text)
|
|
tokenCount := len(words)
|
|
|
|
// Add some estimate for punctuation that might be separate tokens
|
|
punctuationCount := 0
|
|
for _, r := range text {
|
|
if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
|
|
punctuationCount++
|
|
}
|
|
}
|
|
|
|
// Rough estimate: each word is a token, plus half the punctuation as separate tokens
|
|
return tokenCount + (punctuationCount / 2)
|
|
}
|