56 lines
1.7 KiB
Go
56 lines
1.7 KiB
Go
package chunking
|
|
|
|
import (
|
|
"strings"
|
|
"unicode"
|
|
)
|
|
|
|
// TokenEstimator provides a deterministic token estimation suitable for prompt budgeting.
|
|
// The estimator is approximate but stable, isolated, and replaceable.
|
|
type TokenEstimator interface {
|
|
EstimateTokens(text string) int
|
|
}
|
|
|
|
// SimpleTokenEstimator provides a basic deterministic token estimation.
|
|
// This uses a simple heuristic based on word count and punctuation.
|
|
type SimpleTokenEstimator struct{}
|
|
|
|
// NewSimpleTokenEstimator creates a new simple token estimator.
|
|
func NewSimpleTokenEstimator() *SimpleTokenEstimator {
|
|
return &SimpleTokenEstimator{}
|
|
}
|
|
|
|
// EstimateTokens provides a rough estimate of the number of tokens in the given text.
|
|
// This implementation uses a simple heuristic: count words and punctuation as tokens.
|
|
// The estimate is deterministic and stable for the same input text.
|
|
func (e *SimpleTokenEstimator) EstimateTokens(text string) int {
|
|
if text == "" {
|
|
return 0
|
|
}
|
|
|
|
// Simple heuristic: split on whitespace and count non-empty segments
|
|
words := strings.Fields(text)
|
|
tokenCount := len(words)
|
|
|
|
// Add some estimate for punctuation that might be separate tokens
|
|
punctuationCount := 0
|
|
for _, r := range text {
|
|
if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
|
|
punctuationCount++
|
|
}
|
|
}
|
|
|
|
// Rough estimate: each word is a token, plus half the punctuation as separate tokens
|
|
return tokenCount + (punctuationCount / 2)
|
|
}
|
|
|
|
// ConstTokenEstimator returns a constant token count for testing purposes.
|
|
type ConstTokenEstimator struct {
|
|
Tokens int
|
|
}
|
|
|
|
// EstimateTokens returns the configured constant token count.
|
|
func (e *ConstTokenEstimator) EstimateTokens(text string) int {
|
|
return e.Tokens
|
|
}
|