eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Generate and materialize canonical chunk plans

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-17 23:57:59 +00:00
parent 3bfac14397
commit 7844c0a93f
23 changed files with 396 additions and 540 deletions
Download Patch File Download Diff File Expand all files Collapse all files

111
internal/modules/generic/chunk/units/chunker_test.go
View File

@@ -48,72 +48,51 @@ func TestModuleSpecAndRegister(t *testing.T) {
if err != nil {
t.Fatalf("BuildWithRequest(%q) error = %v, want nil", Key, err)
}
result, err := configured.Chunk(context.Background(), contracts.ChunkRequest{Source: testSource(2)})
if err != nil || len(result.Chunks) != 2 {
t.Fatalf("constructed chunker result = %#v, %v; want two chunks", result, err)
result, err := configured.Plan(context.Background(), contracts.ChunkRequest{Source: testSource(2)})
if err != nil || len(result.Plan.Ranges) != 2 {
t.Fatalf("constructed chunker result = %#v, %v; want two ranges", result, err)
}
}
func TestChunkUsesDefaultsForSingleChunk(t *testing.T) {
result, err := newChunker(t, nil).Chunk(context.Background(), contracts.ChunkRequest{Source: testSource(3)})
func TestPlanUsesDefaultsForSingleRange(t *testing.T) {
result, err := newChunker(t, nil).Plan(context.Background(), contracts.ChunkRequest{Source: testSource(3)})
if err != nil {
t.Fatalf("Chunk() error = %v, want nil", err)
t.Fatalf("Plan() error = %v, want nil", err)
}
if got := chunkIDs(result.Chunks); !reflect.DeepEqual(got, []string{"chunk-000001"}) {
t.Fatalf("chunk IDs = %#v, want one stable ID", got)
if result.Plan.SourceDigest != "sha256:source" || !reflect.DeepEqual(result.Plan.Ranges, []source.ChunkRange{{StartUnitID: 1, EndUnitID: 3}}) {
t.Fatalf("Plan = %#v, want source digest and one complete range", result.Plan)
}
chunk := result.Chunks[0]
if chunk.Index != 0 || chunk.SourceID != "source-1" {
t.Fatalf("chunk = %#v, want source and index fields", chunk)
}
if got := unitIDs(chunk.Units); !reflect.DeepEqual(got, []int{1, 2, 3}) {
t.Fatalf("unit IDs = %#v, want all units", got)
}
if chunk.Ref != (source.SourceRef{SourceID: "source-1", StartUnitID: 1, EndUnitID: 3}) {
t.Fatalf("chunk ref = %#v, want source-1:1-3", chunk.Ref)
}
if chunk.MediaType != "application/json" || len(chunk.Content) == 0 {
t.Fatalf("chunk payload = media type %q length %d, want JSON content", chunk.MediaType, len(chunk.Content))
}
if chunk.Metadata["start_unit_id"] != 1 || chunk.Metadata["end_unit_id"] != 3 || chunk.Metadata["unit_count"] != 3 {
t.Fatalf("metadata = %#v, want chunk bounds", chunk.Metadata)
if len(result.Plan.Annotations) != 0 || len(result.Plan.Ranges[0].Annotations) != 0 {
t.Fatalf("Plan annotations = %#v / %#v, want none", result.Plan.Annotations, result.Plan.Ranges[0].Annotations)
}
}
func TestChunkExactBoundaries(t *testing.T) {
result, err := newChunker(t, map[string]any{"max_units": 2}).Chunk(context.Background(), contracts.ChunkRequest{Source: testSource(6)})
func TestPlanExactBoundaries(t *testing.T) {
result, err := newChunker(t, map[string]any{"max_units": 2}).Plan(context.Background(), contracts.ChunkRequest{Source: testSource(6)})
if err != nil {
t.Fatalf("Chunk() error = %v, want nil", err)
t.Fatalf("Plan() error = %v, want nil", err)
}
if got := chunkIDs(result.Chunks); !reflect.DeepEqual(got, []string{"chunk-000001", "chunk-000002", "chunk-000003"}) {
t.Fatalf("chunk IDs = %#v, want stable IDs", got)
}
gotUnits := [][]int{unitIDs(result.Chunks[0].Units), unitIDs(result.Chunks[1].Units), unitIDs(result.Chunks[2].Units)}
wantUnits := [][]int{{1, 2}, {3, 4}, {5, 6}}
if !reflect.DeepEqual(gotUnits, wantUnits) {
t.Fatalf("chunk units = %#v, want %#v", gotUnits, wantUnits)
want := []source.ChunkRange{{StartUnitID: 1, EndUnitID: 2}, {StartUnitID: 3, EndUnitID: 4}, {StartUnitID: 5, EndUnitID: 6}}
if !reflect.DeepEqual(result.Plan.Ranges, want) {
t.Fatalf("ranges = %#v, want %#v", result.Plan.Ranges, want)
}
}
func TestChunkOverlap(t *testing.T) {
result, err := newChunker(t, map[string]any{"max_units": 3, "overlap_units": 1}).Chunk(context.Background(), contracts.ChunkRequest{Source: testSource(7)})
func TestPlanOverlap(t *testing.T) {
result, err := newChunker(t, map[string]any{"max_units": 3, "overlap_units": 1}).Plan(context.Background(), contracts.ChunkRequest{Source: testSource(7)})
if err != nil {
t.Fatalf("Chunk() error = %v, want nil", err)
t.Fatalf("Plan() error = %v, want nil", err)
}
gotUnits := make([][]int, 0, len(result.Chunks))
for _, chunk := range result.Chunks {
gotUnits = append(gotUnits, unitIDs(chunk.Units))
}
wantUnits := [][]int{{1, 2, 3}, {3, 4, 5}, {5, 6, 7}}
if !reflect.DeepEqual(gotUnits, wantUnits) {
t.Fatalf("chunk units = %#v, want %#v", gotUnits, wantUnits)
want := []source.ChunkRange{{StartUnitID: 1, EndUnitID: 3}, {StartUnitID: 3, EndUnitID: 5}, {StartUnitID: 5, EndUnitID: 7}}
if !reflect.DeepEqual(result.Plan.Ranges, want) {
t.Fatalf("ranges = %#v, want %#v", result.Plan.Ranges, want)
}
}
func TestChunkRejectsInvalidOptions(t *testing.T) {
func TestPlanRejectsInvalidOptions(t *testing.T) {
tests := []struct {
name string
options map[string]any
@@ -141,42 +120,36 @@ func TestChunkRejectsInvalidOptions(t *testing.T) {
}
}
func TestChunkRejectsEmptySource(t *testing.T) {
func TestPlanRejectsEmptySource(t *testing.T) {
doc := testSource(1)
doc.Units = nil
_, err := newChunker(t, nil).Chunk(context.Background(), contracts.ChunkRequest{Source: doc})
_, err := newChunker(t, nil).Plan(context.Background(), contracts.ChunkRequest{Source: doc})
if err == nil {
t.Fatal("Chunk() error = nil, want empty source error")
t.Fatal("Plan() error = nil, want empty source error")
}
if !strings.Contains(err.Error(), "generic chunker") || !strings.Contains(err.Error(), "units") {
t.Fatalf("Chunk() error = %q, want empty source context", err.Error())
t.Fatalf("Plan() error = %q, want empty source context", err.Error())
}
}
func TestChunkDefensivelyCopiesUnits(t *testing.T) {
func TestPlanDoesNotRetainSourceUnits(t *testing.T) {
doc := testSource(2)
result, err := newChunker(t, map[string]any{"max_units": 1}).Chunk(context.Background(), contracts.ChunkRequest{Source: doc})
result, err := newChunker(t, map[string]any{"max_units": 1}).Plan(context.Background(), contracts.ChunkRequest{Source: doc})
if err != nil {
t.Fatalf("Chunk() error = %v, want nil", err)
t.Fatalf("Plan() error = %v, want nil", err)
}
if len(result.Chunks) != 2 {
t.Fatalf("len(Chunks) = %d, want 2", len(result.Chunks))
if len(result.Plan.Ranges) != 2 {
t.Fatalf("len(Ranges) = %d, want 2", len(result.Plan.Ranges))
}
doc.Units[0].ID = 99
doc.Units[0].Ref.SourceID = "changed"
doc.Units[0].Metadata["speaker"] = "changed"
if result.Chunks[0].Units[0].ID != 1 {
t.Fatalf("chunk unit ID changed after source mutation: %#v", result.Chunks[0].Units[0])
}
if got := result.Chunks[0].Units[0].Ref.SourceID; got != "source-1" {
t.Fatalf("chunk unit ref changed after source mutation: %#v", result.Chunks[0].Units[0].Ref)
}
if result.Chunks[0].Units[0].Metadata["speaker"] != "speaker-001" {
t.Fatalf("chunk unit metadata changed after source mutation: %#v", result.Chunks[0].Units[0].Metadata)
if result.Plan.Ranges[0].StartUnitID != 1 || result.Plan.Ranges[0].EndUnitID != 1 {
t.Fatalf("first range changed after source mutation: %#v", result.Plan.Ranges[0])
}
}
@@ -214,19 +187,3 @@ func testSource(count int) *source.SourceDocument {
func zeroPad3(value int) string {
return fmt.Sprintf("%03d", value)
}
func chunkIDs(chunks []source.Chunk) []string {
ids := make([]string, 0, len(chunks))
for _, chunk := range chunks {
ids = append(ids, chunk.ID)
}
return ids
}
func unitIDs(units []source.SourceUnit) []int {
ids := make([]int, 0, len(units))
for _, unit := range units {
ids = append(ids, unit.ID)
}
return ids
}