eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Generate and materialize canonical chunk plans

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-17 23:57:59 +00:00
parent 3bfac14397
commit 7844c0a93f
23 changed files with 396 additions and 540 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
internal/modules/integration/concurrent_runner_test.go
View File

@@ -4,7 +4,6 @@ import (
"context"
"encoding/json"
"errors"
"fmt"
"sync/atomic"
"testing"
"time"
@@ -68,12 +67,12 @@ type concurrentChunker struct{}
func (concurrentChunker) Key() string { return concurrentChunkerKey }
func (concurrentChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
func (concurrentChunker) Chunk(_ context.Context, request contracts.ChunkRequest) (contracts.ChunkResult, error) {
chunks := make([]source.Chunk, len(request.Source.Units))
func (concurrentChunker) Plan(_ context.Context, request contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
ranges := make([]source.ChunkRange, len(request.Source.Units))
for i, unit := range request.Source.Units {
chunks[i] = source.Chunk{ID: fmt.Sprintf("%s:chunk:%d", request.Source.ID, i), SourceID: request.Source.ID, Index: i, Ref: unit.Ref, Content: []byte(fmt.Sprintf(`{"unit":%d}`, unit.ID)), MediaType: "application/json", Units: []source.SourceUnit{unit}}
ranges[i] = source.ChunkRange{StartUnitID: unit.ID, EndUnitID: unit.ID}
}
return contracts.ChunkResult{Chunks: chunks}, nil
return contracts.ChunkPlanResult{Plan: source.ChunkPlan{SourceDigest: request.Source.Digest, Ranges: ranges}}, nil
}
type concurrentExtractor struct {

16
internal/modules/integration/dnd_spells_config_test.go
View File

@@ -271,19 +271,9 @@ func (dndSpellsChunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (dndSpellsChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
return contracts.ChunkResult{
Chunks: []source.Chunk{
{
ID: req.Source.ID + ":chunk:0",
SourceID: req.Source.ID,
Index: 0,
Ref: source.SourceRef{SourceID: req.Source.ID, StartUnitID: req.Source.Units[0].ID, EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID},
Content: []byte(`{"units":[1,2,3]}`),
MediaType: "application/json",
Units: append([]source.SourceUnit(nil), req.Source.Units...),
},
},
func (dndSpellsChunker) Plan(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
return contracts.ChunkPlanResult{
Plan: source.ChunkPlan{SourceDigest: req.Source.Digest, Ranges: []source.ChunkRange{{StartUnitID: req.Source.Units[0].ID, EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID}}},
}, nil
}