519 lines
16 KiB
Go
519 lines
16 KiB
Go
package chunking
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/audita/internal/core/schema"
|
|
)
|
|
|
|
// mapTokenEstimator provides deterministic per-segment token counts for tests.
|
|
type mapTokenEstimator struct {
|
|
byText map[string]int
|
|
}
|
|
|
|
func (e *mapTokenEstimator) EstimateTokens(text string) int {
|
|
if e.byText == nil {
|
|
return 0
|
|
}
|
|
if tokens, ok := e.byText[text]; ok {
|
|
return tokens
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func makeSegments(texts []string) []schema.Segment {
|
|
segments := make([]schema.Segment, len(texts))
|
|
for i, text := range texts {
|
|
segments[i] = schema.Segment{
|
|
ID: i + 1,
|
|
Speaker: "DM",
|
|
Start: float64(i * 10),
|
|
End: float64(i*10 + 5),
|
|
Text: text,
|
|
}
|
|
}
|
|
return segments
|
|
}
|
|
|
|
func makeTranscript(segments []schema.Segment) *schema.Transcript {
|
|
return &schema.Transcript{Segments: segments}
|
|
}
|
|
|
|
func intPtr(i int) *int {
|
|
return &i
|
|
}
|
|
|
|
func assertSegmentCoverageAndOrder(t *testing.T, input []schema.Segment, sections []Section) {
|
|
t.Helper()
|
|
|
|
seen := make([]schema.Segment, 0, len(input))
|
|
for _, sec := range sections {
|
|
seen = append(seen, sec.Segments...)
|
|
}
|
|
|
|
if len(seen) != len(input) {
|
|
t.Fatalf("expected %d total segment occurrences, got %d", len(input), len(seen))
|
|
}
|
|
|
|
for i := range input {
|
|
if seen[i].ID != input[i].ID {
|
|
t.Fatalf("segment order mismatch at index %d: got id=%d want id=%d", i, seen[i].ID, input[i].ID)
|
|
}
|
|
}
|
|
}
|
|
|
|
func assertSectionMetadataConsistent(t *testing.T, sections []Section) {
|
|
t.Helper()
|
|
|
|
for i, sec := range sections {
|
|
if sec.Index != i {
|
|
t.Fatalf("section %d: expected index=%d got=%d", i, i, sec.Index)
|
|
}
|
|
if len(sec.Segments) == 0 {
|
|
t.Fatalf("section %d: section must not be empty", i)
|
|
}
|
|
if sec.StartSegmentID != sec.Segments[0].ID {
|
|
t.Fatalf("section %d: start_segment_id mismatch", i)
|
|
}
|
|
if sec.EndSegmentID != sec.Segments[len(sec.Segments)-1].ID {
|
|
t.Fatalf("section %d: end_segment_id mismatch", i)
|
|
}
|
|
}
|
|
}
|
|
|
|
func assertMaxBoundExceptSingletonOversized(t *testing.T, sections []Section, max int) {
|
|
t.Helper()
|
|
|
|
for i, sec := range sections {
|
|
if sec.EstimatedTokens <= max {
|
|
continue
|
|
}
|
|
if len(sec.Segments) != 1 {
|
|
t.Fatalf("section %d exceeds max tokens (%d>%d) with %d segments", i, sec.EstimatedTokens, max, len(sec.Segments))
|
|
}
|
|
}
|
|
}
|
|
|
|
func imbalance(sections []Section) int {
|
|
if len(sections) == 0 {
|
|
return 0
|
|
}
|
|
minTokens := sections[0].EstimatedTokens
|
|
maxTokens := sections[0].EstimatedTokens
|
|
for _, sec := range sections {
|
|
if sec.EstimatedTokens < minTokens {
|
|
minTokens = sec.EstimatedTokens
|
|
}
|
|
if sec.EstimatedTokens > maxTokens {
|
|
maxTokens = sec.EstimatedTokens
|
|
}
|
|
}
|
|
return maxTokens - minTokens
|
|
}
|
|
|
|
func greedyMaxFillSections(segments []schema.Segment, tokens []int, max int) []Section {
|
|
sections := make([]Section, 0)
|
|
var current []schema.Segment
|
|
currentTokens := 0
|
|
|
|
for i, seg := range segments {
|
|
tok := tokens[i]
|
|
if len(current) == 0 {
|
|
current = append(current, seg)
|
|
currentTokens = tok
|
|
continue
|
|
}
|
|
if currentTokens+tok > max {
|
|
sections = append(sections, Section{
|
|
Index: len(sections),
|
|
StartSegmentID: current[0].ID,
|
|
EndSegmentID: current[len(current)-1].ID,
|
|
EstimatedTokens: currentTokens,
|
|
Segments: append([]schema.Segment(nil), current...),
|
|
})
|
|
current = []schema.Segment{seg}
|
|
currentTokens = tok
|
|
continue
|
|
}
|
|
current = append(current, seg)
|
|
currentTokens += tok
|
|
}
|
|
|
|
if len(current) > 0 {
|
|
sections = append(sections, Section{
|
|
Index: len(sections),
|
|
StartSegmentID: current[0].ID,
|
|
EndSegmentID: current[len(current)-1].ID,
|
|
EstimatedTokens: currentTokens,
|
|
Segments: append([]schema.Segment(nil), current...),
|
|
})
|
|
}
|
|
|
|
return sections
|
|
}
|
|
|
|
func TestChunkEmptyTranscript(t *testing.T) {
|
|
chunker := NewChunker(ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 10})
|
|
|
|
sections, err := chunker.ChunkTranscript(nil)
|
|
if err != nil {
|
|
t.Fatalf("ChunkTranscript(nil): %v", err)
|
|
}
|
|
if len(sections) != 0 {
|
|
t.Fatalf("expected 0 sections for nil transcript, got %d", len(sections))
|
|
}
|
|
|
|
sections, err = chunker.ChunkTranscript(makeTranscript(nil))
|
|
if err != nil {
|
|
t.Fatalf("ChunkTranscript(empty): %v", err)
|
|
}
|
|
if len(sections) != 0 {
|
|
t.Fatalf("expected 0 sections for empty transcript, got %d", len(sections))
|
|
}
|
|
}
|
|
|
|
func TestChunkSingleSegment(t *testing.T) {
|
|
segments := makeSegments([]string{"s1"})
|
|
chunker := NewChunkerWithEstimator(
|
|
ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 10},
|
|
&mapTokenEstimator{byText: map[string]int{"s1": 7}},
|
|
)
|
|
|
|
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
|
if err != nil {
|
|
t.Fatalf("ChunkTranscript: %v", err)
|
|
}
|
|
if len(sections) != 1 {
|
|
t.Fatalf("expected 1 section, got %d", len(sections))
|
|
}
|
|
if sections[0].EstimatedTokens != 7 {
|
|
t.Fatalf("expected estimated_tokens=7, got %d", sections[0].EstimatedTokens)
|
|
}
|
|
assertSegmentCoverageAndOrder(t, segments, sections)
|
|
assertSectionMetadataConsistent(t, sections)
|
|
}
|
|
|
|
func TestChunkSingleOversizedSegment(t *testing.T) {
|
|
segments := makeSegments([]string{"big"})
|
|
chunker := NewChunkerWithEstimator(
|
|
ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 5},
|
|
&mapTokenEstimator{byText: map[string]int{"big": 120}},
|
|
)
|
|
|
|
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
|
if err != nil {
|
|
t.Fatalf("ChunkTranscript: %v", err)
|
|
}
|
|
if len(sections) != 1 {
|
|
t.Fatalf("expected 1 section, got %d", len(sections))
|
|
}
|
|
if sections[0].EstimatedTokens != 120 {
|
|
t.Fatalf("expected oversized singleton section, got %d", sections[0].EstimatedTokens)
|
|
}
|
|
assertMaxBoundExceptSingletonOversized(t, sections, 50)
|
|
}
|
|
|
|
func TestChunkTotalBelowMaxSingleSection(t *testing.T) {
|
|
segments := makeSegments([]string{"a", "b", "c"})
|
|
chunker := NewChunkerWithEstimator(
|
|
ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 5},
|
|
&ConstTokenEstimator{Tokens: 10},
|
|
)
|
|
|
|
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
|
if err != nil {
|
|
t.Fatalf("ChunkTranscript: %v", err)
|
|
}
|
|
if len(sections) != 1 {
|
|
t.Fatalf("expected 1 section, got %d", len(sections))
|
|
}
|
|
if sections[0].EstimatedTokens != 30 {
|
|
t.Fatalf("expected 30 section tokens, got %d", sections[0].EstimatedTokens)
|
|
}
|
|
assertSegmentCoverageAndOrder(t, segments, sections)
|
|
}
|
|
|
|
func TestChunkTotalExactlyDivisibleByMax(t *testing.T) {
|
|
segments := makeSegments([]string{"a", "b", "c", "d"})
|
|
chunker := NewChunkerWithEstimator(
|
|
ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 1},
|
|
&ConstTokenEstimator{Tokens: 5},
|
|
)
|
|
|
|
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
|
if err != nil {
|
|
t.Fatalf("ChunkTranscript: %v", err)
|
|
}
|
|
if len(sections) != 2 {
|
|
t.Fatalf("expected 2 sections, got %d", len(sections))
|
|
}
|
|
if sections[0].EstimatedTokens != 10 || sections[1].EstimatedTokens != 10 {
|
|
t.Fatalf("expected [10,10] tokens, got [%d,%d]", sections[0].EstimatedTokens, sections[1].EstimatedTokens)
|
|
}
|
|
assertSegmentCoverageAndOrder(t, segments, sections)
|
|
assertMaxBoundExceptSingletonOversized(t, sections, 10)
|
|
}
|
|
|
|
func TestChunkTotalNotDivisibleByMax(t *testing.T) {
|
|
segments := makeSegments([]string{"a", "b", "c", "d", "e"})
|
|
chunker := NewChunkerWithEstimator(
|
|
ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 1},
|
|
&ConstTokenEstimator{Tokens: 5},
|
|
)
|
|
|
|
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
|
if err != nil {
|
|
t.Fatalf("ChunkTranscript: %v", err)
|
|
}
|
|
if len(sections) != 3 {
|
|
t.Fatalf("expected 3 sections, got %d", len(sections))
|
|
}
|
|
if sections[0].EstimatedTokens != 10 || sections[1].EstimatedTokens != 10 || sections[2].EstimatedTokens != 5 {
|
|
t.Fatalf("expected [10,10,5] tokens, got [%d,%d,%d]", sections[0].EstimatedTokens, sections[1].EstimatedTokens, sections[2].EstimatedTokens)
|
|
}
|
|
assertSegmentCoverageAndOrder(t, segments, sections)
|
|
assertMaxBoundExceptSingletonOversized(t, sections, 10)
|
|
}
|
|
|
|
func TestChunkTargetSectionsPrecedenceAndSuccess(t *testing.T) {
|
|
segments := makeSegments([]string{"1", "2", "3", "4", "5", "6", "7", "8", "9", "10"})
|
|
target := 3
|
|
chunker := NewChunkerWithEstimator(
|
|
ChunkingConfig{MaxSectionTokens: 200, MinSectionTokens: 1, TargetSections: &target},
|
|
&ConstTokenEstimator{Tokens: 10},
|
|
)
|
|
|
|
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
|
if err != nil {
|
|
t.Fatalf("ChunkTranscript: %v", err)
|
|
}
|
|
if len(sections) != target {
|
|
t.Fatalf("expected %d sections from explicit target, got %d", target, len(sections))
|
|
}
|
|
assertSegmentCoverageAndOrder(t, segments, sections)
|
|
assertSectionMetadataConsistent(t, sections)
|
|
}
|
|
|
|
func TestChunkTargetSectionsImpossibleTooMany(t *testing.T) {
|
|
segments := makeSegments([]string{"1", "2", "3", "4", "5"})
|
|
target := 10
|
|
chunker := NewChunker(ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 1, TargetSections: &target})
|
|
|
|
_, err := chunker.ChunkTranscript(makeTranscript(segments))
|
|
if err == nil {
|
|
t.Fatal("expected error for impossible target_sections")
|
|
}
|
|
if !strings.Contains(err.Error(), "cannot have more sections than segments") {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestChunkTargetSectionsImpossibleTooFew(t *testing.T) {
|
|
segments := makeSegments([]string{"1", "2", "3"})
|
|
target := 1
|
|
chunker := NewChunkerWithEstimator(
|
|
ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 1, TargetSections: &target},
|
|
&ConstTokenEstimator{Tokens: 30},
|
|
)
|
|
|
|
_, err := chunker.ChunkTranscript(makeTranscript(segments))
|
|
if err == nil {
|
|
t.Fatal("expected error for impossible target_sections")
|
|
}
|
|
if !strings.Contains(err.Error(), "need at least 3 sections") {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestChunkManySmallSegmentsBalanced(t *testing.T) {
|
|
texts := []string{"1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11"}
|
|
segments := makeSegments(texts)
|
|
chunker := NewChunkerWithEstimator(
|
|
ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 1},
|
|
&ConstTokenEstimator{Tokens: 10},
|
|
)
|
|
|
|
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
|
if err != nil {
|
|
t.Fatalf("ChunkTranscript: %v", err)
|
|
}
|
|
if len(sections) != 3 {
|
|
t.Fatalf("expected 3 sections, got %d", len(sections))
|
|
}
|
|
if sections[0].EstimatedTokens != 40 || sections[1].EstimatedTokens != 40 || sections[2].EstimatedTokens != 30 {
|
|
t.Fatalf("expected [40,40,30], got [%d,%d,%d]", sections[0].EstimatedTokens, sections[1].EstimatedTokens, sections[2].EstimatedTokens)
|
|
}
|
|
assertSegmentCoverageAndOrder(t, segments, sections)
|
|
assertMaxBoundExceptSingletonOversized(t, sections, 50)
|
|
}
|
|
|
|
func TestChunkMixedLargeAndSmallSegments(t *testing.T) {
|
|
segments := makeSegments([]string{"big1", "s1", "s2", "s3", "big2", "s4"})
|
|
estimator := &mapTokenEstimator{byText: map[string]int{
|
|
"big1": 120,
|
|
"s1": 10,
|
|
"s2": 10,
|
|
"s3": 10,
|
|
"big2": 120,
|
|
"s4": 10,
|
|
}}
|
|
chunker := NewChunkerWithEstimator(ChunkingConfig{MaxSectionTokens: 100, MinSectionTokens: 1}, estimator)
|
|
|
|
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
|
if err != nil {
|
|
t.Fatalf("ChunkTranscript: %v", err)
|
|
}
|
|
if len(sections) != 4 {
|
|
t.Fatalf("expected 4 sections, got %d", len(sections))
|
|
}
|
|
if len(sections[0].Segments) != 1 || sections[0].Segments[0].Text != "big1" {
|
|
t.Fatalf("expected first oversized segment in singleton section, got %+v", sections[0].Segments)
|
|
}
|
|
if len(sections[2].Segments) != 1 || sections[2].Segments[0].Text != "big2" {
|
|
t.Fatalf("expected second oversized segment in singleton section, got %+v", sections[2].Segments)
|
|
}
|
|
assertSegmentCoverageAndOrder(t, segments, sections)
|
|
assertMaxBoundExceptSingletonOversized(t, sections, 100)
|
|
}
|
|
|
|
func TestChunkDeterministicOrdering(t *testing.T) {
|
|
segments := makeSegments([]string{"a", "b", "c", "d", "e", "f"})
|
|
chunkerCfg := ChunkingConfig{MaxSectionTokens: 15, MinSectionTokens: 1}
|
|
estimator := &ConstTokenEstimator{Tokens: 5}
|
|
|
|
var first []Section
|
|
for i := 0; i < 5; i++ {
|
|
chunker := NewChunkerWithEstimator(chunkerCfg, estimator)
|
|
sections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
|
if err != nil {
|
|
t.Fatalf("iteration %d: %v", i, err)
|
|
}
|
|
if i == 0 {
|
|
first = sections
|
|
continue
|
|
}
|
|
if len(sections) != len(first) {
|
|
t.Fatalf("iteration %d: section count mismatch (%d vs %d)", i, len(sections), len(first))
|
|
}
|
|
for j := range sections {
|
|
if sections[j].Index != first[j].Index ||
|
|
sections[j].StartSegmentID != first[j].StartSegmentID ||
|
|
sections[j].EndSegmentID != first[j].EndSegmentID ||
|
|
sections[j].EstimatedTokens != first[j].EstimatedTokens ||
|
|
len(sections[j].Segments) != len(first[j].Segments) {
|
|
t.Fatalf("iteration %d section %d mismatch", i, j)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestChunkNoMutationOfInput(t *testing.T) {
|
|
segments := makeSegments([]string{"original one", "original two"})
|
|
transcript := makeTranscript(segments)
|
|
original := make([]string, len(transcript.Segments))
|
|
for i := range transcript.Segments {
|
|
original[i] = transcript.Segments[i].Text
|
|
}
|
|
|
|
chunker := NewChunker(ChunkingConfig{MaxSectionTokens: 50, MinSectionTokens: 1})
|
|
if _, err := chunker.ChunkTranscript(transcript); err != nil {
|
|
t.Fatalf("ChunkTranscript: %v", err)
|
|
}
|
|
for i := range transcript.Segments {
|
|
if transcript.Segments[i].Text != original[i] {
|
|
t.Fatalf("segment %d mutated", i)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestChunkConfigValidation(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
config ChunkingConfig
|
|
errContains string
|
|
}{
|
|
{
|
|
name: "zero max tokens",
|
|
config: ChunkingConfig{MaxSectionTokens: 0, MinSectionTokens: 1},
|
|
errContains: "max_section_tokens must be positive",
|
|
},
|
|
{
|
|
name: "negative max tokens",
|
|
config: ChunkingConfig{MaxSectionTokens: -1, MinSectionTokens: 1},
|
|
errContains: "max_section_tokens must be positive",
|
|
},
|
|
{
|
|
name: "negative min tokens",
|
|
config: ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: -1},
|
|
errContains: "min_section_tokens must be non-negative",
|
|
},
|
|
{
|
|
name: "min exceeds max",
|
|
config: ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 11},
|
|
errContains: "min_section_tokens (11) cannot exceed max_section_tokens (10)",
|
|
},
|
|
{
|
|
name: "zero target sections",
|
|
config: ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 1, TargetSections: intPtr(0)},
|
|
errContains: "target_sections must be positive",
|
|
},
|
|
{
|
|
name: "negative target sections",
|
|
config: ChunkingConfig{MaxSectionTokens: 10, MinSectionTokens: 1, TargetSections: intPtr(-1)},
|
|
errContains: "target_sections must be positive",
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
chunker := NewChunker(tt.config)
|
|
_, err := chunker.ChunkTranscript(makeTranscript(makeSegments([]string{"x"})))
|
|
if err == nil {
|
|
t.Fatalf("expected error containing %q", tt.errContains)
|
|
}
|
|
if !strings.Contains(err.Error(), tt.errContains) {
|
|
t.Fatalf("expected error containing %q, got %q", tt.errContains, err.Error())
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestChunkBalancedAlgorithmBeatsGreedyMaxFillOnUnevenTranscript(t *testing.T) {
|
|
segments := makeSegments([]string{"s1", "s2", "s3", "s4", "s5", "s6"})
|
|
tokenMap := map[string]int{
|
|
"s1": 50,
|
|
"s2": 10,
|
|
"s3": 10,
|
|
"s4": 10,
|
|
"s5": 10,
|
|
"s6": 10,
|
|
}
|
|
estimator := &mapTokenEstimator{byText: tokenMap}
|
|
chunker := NewChunkerWithEstimator(ChunkingConfig{MaxSectionTokens: 80, MinSectionTokens: 1}, estimator)
|
|
|
|
balancedSections, err := chunker.ChunkTranscript(makeTranscript(segments))
|
|
if err != nil {
|
|
t.Fatalf("ChunkTranscript: %v", err)
|
|
}
|
|
|
|
tokens := make([]int, 0, len(segments))
|
|
for _, seg := range segments {
|
|
tokens = append(tokens, tokenMap[seg.Text])
|
|
}
|
|
greedySections := greedyMaxFillSections(segments, tokens, 80)
|
|
|
|
balancedImbalance := imbalance(balancedSections)
|
|
greedyImbalance := imbalance(greedySections)
|
|
if balancedImbalance >= greedyImbalance {
|
|
t.Fatalf(
|
|
"expected balanced chunking to improve over greedy max-fill; balanced=%d greedy=%d",
|
|
balancedImbalance,
|
|
greedyImbalance,
|
|
)
|
|
}
|
|
|
|
assertSegmentCoverageAndOrder(t, segments, balancedSections)
|
|
assertMaxBoundExceptSingletonOversized(t, balancedSections, 80)
|
|
}
|