package normalization

import (
	"strings"
	"unicode"
)

// SimpleTokenEstimator provides a basic deterministic token estimation.
// This is a placeholder that can be replaced with a more sophisticated estimator later.
type SimpleTokenEstimator struct{}

// EstimateTokens provides a rough estimate of the number of tokens in the given text.
// This implementation uses a simple heuristic: count words and punctuation as tokens.
func (e *SimpleTokenEstimator) EstimateTokens(text string) int {
	if text == "" {
		return 0
	}

	// Simple heuristic: split on whitespace and count non-empty segments
	words := strings.Fields(text)
	tokenCount := len(words)

	// Add some estimate for punctuation that might be separate tokens
	punctuationCount := 0
	for _, r := range text {
		if unicode.IsPunct(r) && r != '\'' && r != '-' && r != '_' {
			punctuationCount++
		}
	}

	// Rough estimate: each word is a token, plus half the punctuation as separate tokens
	return tokenCount + (punctuationCount / 2)
}