207 lines
6.9 KiB
Go
207 lines
6.9 KiB
Go
package units
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
|
|
)
|
|
|
|
func TestModuleSpecAndRegister(t *testing.T) {
|
|
want := pipeline.ModuleSpec{
|
|
Key: Key,
|
|
Stage: pipeline.StageChunk,
|
|
ExecutionClass: contracts.ExecutionClassDeterministic,
|
|
Provides: []string{"chunks"},
|
|
}
|
|
if got := ModuleSpec(); !reflect.DeepEqual(got, want) {
|
|
t.Fatalf("ModuleSpec() = %#v, want %#v", got, want)
|
|
}
|
|
|
|
registry := pipeline.NewChunkerRegistry()
|
|
if err := Register(registry); err != nil {
|
|
t.Fatalf("Register() error = %v, want nil", err)
|
|
}
|
|
spec, ok := registry.Spec(Key)
|
|
if !ok {
|
|
t.Fatalf("Spec(%q) ok = false, want true", Key)
|
|
}
|
|
if !reflect.DeepEqual(spec, want) {
|
|
t.Fatalf("registered spec = %#v, want %#v", spec, want)
|
|
}
|
|
chunker, err := registry.Build(Key)
|
|
if err != nil {
|
|
t.Fatalf("Build(%q) error = %v, want nil", Key, err)
|
|
}
|
|
if chunker.Key() != Key {
|
|
t.Fatalf("Key() = %q, want %q", chunker.Key(), Key)
|
|
}
|
|
if slots := chunker.ReferenceSlots(); len(slots) != 0 {
|
|
t.Fatalf("ReferenceSlots() = %#v, want none", slots)
|
|
}
|
|
configured, err := registry.BuildWithRequest(Key, pipeline.BuildRequest{Options: map[string]any{"max_units": 1}})
|
|
if err != nil {
|
|
t.Fatalf("BuildWithRequest(%q) error = %v, want nil", Key, err)
|
|
}
|
|
result, err := configured.Plan(context.Background(), contracts.ChunkRequest{Source: testSource(2)})
|
|
if err != nil || len(result.Plan.Ranges) != 2 {
|
|
t.Fatalf("constructed chunker result = %#v, %v; want two ranges", result, err)
|
|
}
|
|
}
|
|
|
|
func TestPlanUsesDefaultsForSingleRange(t *testing.T) {
|
|
result, err := newChunker(t, nil).Plan(context.Background(), contracts.ChunkRequest{Source: testSource(3)})
|
|
if err != nil {
|
|
t.Fatalf("Plan() error = %v, want nil", err)
|
|
}
|
|
|
|
if result.Plan.SourceDigest != "sha256:source" || !reflect.DeepEqual(result.Plan.Ranges, []source.ChunkRange{{StartUnitID: 1, EndUnitID: 3}}) {
|
|
t.Fatalf("Plan = %#v, want source digest and one complete range", result.Plan)
|
|
}
|
|
if len(result.Plan.Annotations) != 0 || len(result.Plan.Ranges[0].Annotations) != 0 {
|
|
t.Fatalf("Plan annotations = %#v / %#v, want none", result.Plan.Annotations, result.Plan.Ranges[0].Annotations)
|
|
}
|
|
}
|
|
|
|
func TestPlanExactBoundaries(t *testing.T) {
|
|
result, err := newChunker(t, map[string]any{"max_units": 2}).Plan(context.Background(), contracts.ChunkRequest{Source: testSource(6)})
|
|
if err != nil {
|
|
t.Fatalf("Plan() error = %v, want nil", err)
|
|
}
|
|
|
|
want := []source.ChunkRange{{StartUnitID: 1, EndUnitID: 2}, {StartUnitID: 3, EndUnitID: 4}, {StartUnitID: 5, EndUnitID: 6}}
|
|
if !reflect.DeepEqual(result.Plan.Ranges, want) {
|
|
t.Fatalf("ranges = %#v, want %#v", result.Plan.Ranges, want)
|
|
}
|
|
}
|
|
|
|
func TestPlanOverlap(t *testing.T) {
|
|
result, err := newChunker(t, map[string]any{"max_units": 3, "overlap_units": 1}).Plan(context.Background(), contracts.ChunkRequest{Source: testSource(7)})
|
|
if err != nil {
|
|
t.Fatalf("Plan() error = %v, want nil", err)
|
|
}
|
|
|
|
want := []source.ChunkRange{{StartUnitID: 1, EndUnitID: 3}, {StartUnitID: 3, EndUnitID: 5}, {StartUnitID: 5, EndUnitID: 7}}
|
|
if !reflect.DeepEqual(result.Plan.Ranges, want) {
|
|
t.Fatalf("ranges = %#v, want %#v", result.Plan.Ranges, want)
|
|
}
|
|
}
|
|
|
|
func TestPlanRejectsInvalidOptions(t *testing.T) {
|
|
tests := []struct {
|
|
name string
|
|
options map[string]any
|
|
want string
|
|
}{
|
|
{name: "max wrong type", options: map[string]any{"max_units": "2"}, want: "max_units"},
|
|
{name: "max fractional", options: map[string]any{"max_units": 1.5}, want: "integer"},
|
|
{name: "max zero", options: map[string]any{"max_units": 0}, want: "positive"},
|
|
{name: "overlap negative", options: map[string]any{"overlap_units": -1}, want: "non-negative"},
|
|
{name: "overlap too large", options: map[string]any{"max_units": 2, "overlap_units": 2}, want: "less than"},
|
|
{name: "json number", options: map[string]any{"max_units": json.Number("bad")}, want: "integer"},
|
|
{name: "unknown", options: map[string]any{"unexpected": true}, want: "unknown option"},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
t.Run(test.name, func(t *testing.T) {
|
|
_, err := DecodeOptions(test.options)
|
|
if err == nil {
|
|
t.Fatal("DecodeOptions() error = nil, want error")
|
|
}
|
|
if !strings.Contains(err.Error(), "generic chunker") || !strings.Contains(err.Error(), test.want) {
|
|
t.Fatalf("DecodeOptions() error = %q, want module context and %q", err.Error(), test.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDecodeOptionsRejectsNonNativeIntegers(t *testing.T) {
|
|
for _, value := range []any{
|
|
int8(2), int16(2), int32(2), int64(2),
|
|
uint(2), uint8(2), uint16(2), uint32(2), uint64(2),
|
|
float32(2), float64(2), json.Number("2"),
|
|
} {
|
|
_, err := DecodeOptions(map[string]any{"max_units": value})
|
|
if err == nil {
|
|
t.Fatalf("DecodeOptions(max_units: %T) error = nil, want error", value)
|
|
}
|
|
if !strings.Contains(err.Error(), "max_units must be an integer") {
|
|
t.Fatalf("DecodeOptions(max_units: %T) error = %q, want integer type error", value, err)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestPlanRejectsEmptySource(t *testing.T) {
|
|
doc := testSource(1)
|
|
doc.Units = nil
|
|
|
|
_, err := newChunker(t, nil).Plan(context.Background(), contracts.ChunkRequest{Source: doc})
|
|
if err == nil {
|
|
t.Fatal("Plan() error = nil, want empty source error")
|
|
}
|
|
if !strings.Contains(err.Error(), "generic chunker") || !strings.Contains(err.Error(), "units") {
|
|
t.Fatalf("Plan() error = %q, want empty source context", err.Error())
|
|
}
|
|
}
|
|
|
|
func TestPlanDoesNotRetainSourceUnits(t *testing.T) {
|
|
doc := testSource(2)
|
|
|
|
result, err := newChunker(t, map[string]any{"max_units": 1}).Plan(context.Background(), contracts.ChunkRequest{Source: doc})
|
|
if err != nil {
|
|
t.Fatalf("Plan() error = %v, want nil", err)
|
|
}
|
|
if len(result.Plan.Ranges) != 2 {
|
|
t.Fatalf("len(Ranges) = %d, want 2", len(result.Plan.Ranges))
|
|
}
|
|
|
|
doc.Units[0].ID = 99
|
|
doc.Units[0].Ref.SourceID = "changed"
|
|
doc.Units[0].Metadata["speaker"] = "changed"
|
|
|
|
if result.Plan.Ranges[0].StartUnitID != 1 || result.Plan.Ranges[0].EndUnitID != 1 {
|
|
t.Fatalf("first range changed after source mutation: %#v", result.Plan.Ranges[0])
|
|
}
|
|
}
|
|
|
|
func newChunker(t *testing.T, rawOptions map[string]any) *Chunker {
|
|
t.Helper()
|
|
options, err := DecodeOptions(rawOptions)
|
|
if err != nil {
|
|
t.Fatalf("DecodeOptions() error = %v, want nil", err)
|
|
}
|
|
return New(options)
|
|
}
|
|
|
|
func testSource(count int) *source.SourceDocument {
|
|
units := make([]source.SourceUnit, 0, count)
|
|
for i := 1; i <= count; i++ {
|
|
units = append(units, source.SourceUnit{
|
|
ID: i,
|
|
Kind: "unit",
|
|
Text: "Text for " + zeroPad3(i),
|
|
Ref: source.SourceRef{SourceID: "source-1", StartUnitID: i, EndUnitID: i},
|
|
Metadata: map[string]any{
|
|
"speaker": "speaker-" + zeroPad3(i),
|
|
},
|
|
})
|
|
}
|
|
return &source.SourceDocument{
|
|
ID: "source-1",
|
|
Kind: "document",
|
|
Format: "text/plain",
|
|
Digest: "sha256:source",
|
|
Units: units,
|
|
}
|
|
}
|
|
|
|
func zeroPad3(value int) string {
|
|
return fmt.Sprintf("%03d", value)
|
|
}
|