package units import ( "context" "encoding/json" "fmt" "reflect" "strings" "testing" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" ) func TestModuleSpecAndRegister(t *testing.T) { want := pipeline.ModuleSpec{ Key: Key, Stage: pipeline.StageChunk, ExecutionClass: contracts.ExecutionClassDeterministic, Provides: []string{"chunks"}, } if got := ModuleSpec(); !reflect.DeepEqual(got, want) { t.Fatalf("ModuleSpec() = %#v, want %#v", got, want) } registry := pipeline.NewChunkerRegistry() if err := Register(registry); err != nil { t.Fatalf("Register() error = %v, want nil", err) } spec, ok := registry.Spec(Key) if !ok { t.Fatalf("Spec(%q) ok = false, want true", Key) } if !reflect.DeepEqual(spec, want) { t.Fatalf("registered spec = %#v, want %#v", spec, want) } chunker, err := registry.Build(Key) if err != nil { t.Fatalf("Build(%q) error = %v, want nil", Key, err) } if chunker.Key() != Key { t.Fatalf("Key() = %q, want %q", chunker.Key(), Key) } if slots := chunker.ReferenceSlots(); len(slots) != 0 { t.Fatalf("ReferenceSlots() = %#v, want none", slots) } configured, err := registry.BuildWithRequest(Key, pipeline.BuildRequest{Options: map[string]any{"max_units": 1}}) if err != nil { t.Fatalf("BuildWithRequest(%q) error = %v, want nil", Key, err) } result, err := configured.Plan(context.Background(), contracts.ChunkRequest{Source: testSource(2)}) if err != nil || len(result.Plan.Ranges) != 2 { t.Fatalf("constructed chunker result = %#v, %v; want two ranges", result, err) } } func TestPlanUsesDefaultsForSingleRange(t *testing.T) { result, err := newChunker(t, nil).Plan(context.Background(), contracts.ChunkRequest{Source: testSource(3)}) if err != nil { t.Fatalf("Plan() error = %v, want nil", err) } if result.Plan.SourceDigest != "sha256:source" || !reflect.DeepEqual(result.Plan.Ranges, []source.ChunkRange{{StartUnitID: 1, EndUnitID: 3}}) { t.Fatalf("Plan = %#v, want source digest and one complete range", result.Plan) } if len(result.Plan.Annotations) != 0 || len(result.Plan.Ranges[0].Annotations) != 0 { t.Fatalf("Plan annotations = %#v / %#v, want none", result.Plan.Annotations, result.Plan.Ranges[0].Annotations) } } func TestPlanExactBoundaries(t *testing.T) { result, err := newChunker(t, map[string]any{"max_units": 2}).Plan(context.Background(), contracts.ChunkRequest{Source: testSource(6)}) if err != nil { t.Fatalf("Plan() error = %v, want nil", err) } want := []source.ChunkRange{{StartUnitID: 1, EndUnitID: 2}, {StartUnitID: 3, EndUnitID: 4}, {StartUnitID: 5, EndUnitID: 6}} if !reflect.DeepEqual(result.Plan.Ranges, want) { t.Fatalf("ranges = %#v, want %#v", result.Plan.Ranges, want) } } func TestPlanOverlap(t *testing.T) { result, err := newChunker(t, map[string]any{"max_units": 3, "overlap_units": 1}).Plan(context.Background(), contracts.ChunkRequest{Source: testSource(7)}) if err != nil { t.Fatalf("Plan() error = %v, want nil", err) } want := []source.ChunkRange{{StartUnitID: 1, EndUnitID: 3}, {StartUnitID: 3, EndUnitID: 5}, {StartUnitID: 5, EndUnitID: 7}} if !reflect.DeepEqual(result.Plan.Ranges, want) { t.Fatalf("ranges = %#v, want %#v", result.Plan.Ranges, want) } } func TestPlanRejectsInvalidOptions(t *testing.T) { tests := []struct { name string options map[string]any want string }{ {name: "max wrong type", options: map[string]any{"max_units": "2"}, want: "max_units"}, {name: "max fractional", options: map[string]any{"max_units": 1.5}, want: "integer"}, {name: "max zero", options: map[string]any{"max_units": 0}, want: "positive"}, {name: "overlap negative", options: map[string]any{"overlap_units": -1}, want: "non-negative"}, {name: "overlap too large", options: map[string]any{"max_units": 2, "overlap_units": 2}, want: "less than"}, {name: "json number", options: map[string]any{"max_units": json.Number("bad")}, want: "integer"}, {name: "unknown", options: map[string]any{"unexpected": true}, want: "unknown option"}, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { _, err := DecodeOptions(test.options) if err == nil { t.Fatal("DecodeOptions() error = nil, want error") } if !strings.Contains(err.Error(), "generic chunker") || !strings.Contains(err.Error(), test.want) { t.Fatalf("DecodeOptions() error = %q, want module context and %q", err.Error(), test.want) } }) } } func TestDecodeOptionsRejectsNonNativeIntegers(t *testing.T) { for _, value := range []any{ int8(2), int16(2), int32(2), int64(2), uint(2), uint8(2), uint16(2), uint32(2), uint64(2), float32(2), float64(2), json.Number("2"), } { _, err := DecodeOptions(map[string]any{"max_units": value}) if err == nil { t.Fatalf("DecodeOptions(max_units: %T) error = nil, want error", value) } if !strings.Contains(err.Error(), "max_units must be an integer") { t.Fatalf("DecodeOptions(max_units: %T) error = %q, want integer type error", value, err) } } } func TestPlanRejectsEmptySource(t *testing.T) { doc := testSource(1) doc.Units = nil _, err := newChunker(t, nil).Plan(context.Background(), contracts.ChunkRequest{Source: doc}) if err == nil { t.Fatal("Plan() error = nil, want empty source error") } if !strings.Contains(err.Error(), "generic chunker") || !strings.Contains(err.Error(), "units") { t.Fatalf("Plan() error = %q, want empty source context", err.Error()) } } func TestPlanDoesNotRetainSourceUnits(t *testing.T) { doc := testSource(2) result, err := newChunker(t, map[string]any{"max_units": 1}).Plan(context.Background(), contracts.ChunkRequest{Source: doc}) if err != nil { t.Fatalf("Plan() error = %v, want nil", err) } if len(result.Plan.Ranges) != 2 { t.Fatalf("len(Ranges) = %d, want 2", len(result.Plan.Ranges)) } doc.Units[0].ID = 99 doc.Units[0].Ref.SourceID = "changed" doc.Units[0].Metadata["speaker"] = "changed" if result.Plan.Ranges[0].StartUnitID != 1 || result.Plan.Ranges[0].EndUnitID != 1 { t.Fatalf("first range changed after source mutation: %#v", result.Plan.Ranges[0]) } } func newChunker(t *testing.T, rawOptions map[string]any) *Chunker { t.Helper() options, err := DecodeOptions(rawOptions) if err != nil { t.Fatalf("DecodeOptions() error = %v, want nil", err) } return New(options) } func testSource(count int) *source.SourceDocument { units := make([]source.SourceUnit, 0, count) for i := 1; i <= count; i++ { units = append(units, source.SourceUnit{ ID: i, Kind: "unit", Text: "Text for " + zeroPad3(i), Ref: source.SourceRef{SourceID: "source-1", StartUnitID: i, EndUnitID: i}, Metadata: map[string]any{ "speaker": "speaker-" + zeroPad3(i), }, }) } return &source.SourceDocument{ ID: "source-1", Kind: "document", Format: "text/plain", Digest: "sha256:source", Units: units, } } func zeroPad3(value int) string { return fmt.Sprintf("%03d", value) }