eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Generate and materialize canonical chunk plans

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-17 23:57:59 +00:00
parent 3bfac14397
commit 7844c0a93f
23 changed files with 396 additions and 540 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
internal/framework/contracts/contracts.go
View File

@@ -146,15 +146,15 @@ type ChunkRequest struct {
Metadata map[string]any `json:"metadata,omitempty"`
}
type ChunkResult struct {
Chunks []source.Chunk `json:"chunks"`
Warnings []Warning `json:"warnings,omitempty"`
type ChunkPlanResult struct {
Plan source.ChunkPlan `json:"plan"`
Warnings []Warning `json:"warnings,omitempty"`
}
type Chunker interface {
Key() string
ReferenceSlots() []ReferenceSlot
Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error)
Plan(ctx context.Context, req ChunkRequest) (ChunkPlanResult, error)
}
const (

63
internal/framework/contracts/contracts_test.go
View File

@@ -45,7 +45,7 @@ func TestFakeExtractorReturnsTypedOutput(t *testing.T) {
}
}
func TestFakeChunkerReturnsSourceChunks(t *testing.T) {
func TestFakeChunkerReturnsSourcePlan(t *testing.T) {
doc := &source.SourceDocument{
ID: "source-1",
Kind: "document",
@@ -57,36 +57,19 @@ func TestFakeChunkerReturnsSourceChunks(t *testing.T) {
}
chunker := fakeChunker{key: "generic-chunker"}
result, err := chunker.Chunk(context.Background(), ChunkRequest{Source: doc})
result, err := chunker.Plan(context.Background(), ChunkRequest{Source: doc})
if err != nil {
t.Fatalf("Chunk() error = %v, want nil", err)
t.Fatalf("Plan() error = %v, want nil", err)
}
if chunker.Key() != "generic-chunker" {
t.Fatalf("Key() = %q, want generic-chunker", chunker.Key())
}
if len(result.Chunks) != 1 {
t.Fatalf("len(Chunks) = %d, want 1", len(result.Chunks))
if len(result.Plan.Ranges) != 1 {
t.Fatalf("len(Ranges) = %d, want 1", len(result.Plan.Ranges))
}
chunk := result.Chunks[0]
if chunk.ID != "source-1:chunk:0" {
t.Fatalf("source.Chunk.ID = %q, want source-1:chunk:0", chunk.ID)
}
if chunk.SourceID != doc.ID {
t.Fatalf("source.Chunk.SourceID = %q, want %q", chunk.SourceID, doc.ID)
}
if chunk.Index != 0 {
t.Fatalf("source.Chunk.Index = %d, want 0", chunk.Index)
}
if chunk.Ref.StartUnitID != 1 || chunk.Ref.EndUnitID != 1 {
t.Fatalf("source.Chunk.Ref = %#v, want source-1:1-1", chunk.Ref)
}
if chunk.MediaType != "application/json" || string(chunk.Content) != `{"units":[{"id":1,"kind":"section","text":"Source text."}]}` {
t.Fatalf("source.Chunk payload = %q %s, want JSON units", chunk.MediaType, chunk.Content)
}
if len(chunk.Units) != 1 {
t.Fatalf("len(source.Chunk.Units) = %d, want 1", len(chunk.Units))
if result.Plan.SourceDigest != doc.Digest || result.Plan.Ranges[0].StartUnitID != 1 || result.Plan.Ranges[0].EndUnitID != 1 {
t.Fatalf("Plan = %#v, want source digest and unit range", result.Plan)
}
}
@@ -102,8 +85,8 @@ func TestFakeChunkerReceivesPerRunContext(t *testing.T) {
}
chunker := &recordingChunker{key: "llm-chunker"}
if _, err := chunker.Chunk(context.Background(), ChunkRequest{Source: doc, SessionID: "session", LLMProfile: "profile"}); err != nil {
t.Fatalf("Chunk() error = %v, want nil", err)
if _, err := chunker.Plan(context.Background(), ChunkRequest{Source: doc, SessionID: "session", LLMProfile: "profile"}); err != nil {
t.Fatalf("Plan() error = %v, want nil", err)
}
if chunker.request.SessionID != "session" || chunker.request.LLMProfile != "profile" {
t.Fatalf("ChunkRequest = %#v, want per-run session and profile", chunker.request)
@@ -446,22 +429,14 @@ func (chunker fakeChunker) ReferenceSlots() []ReferenceSlot {
return nil
}
func (chunker fakeChunker) Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error) {
return ChunkResult{
Chunks: []source.Chunk{
{
ID: req.Source.ID + ":chunk:0",
SourceID: req.Source.ID,
Index: 0,
Ref: source.SourceRef{
SourceID: req.Source.ID,
StartUnitID: req.Source.Units[0].ID,
EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID,
},
Content: []byte(`{"units":[{"id":1,"kind":"section","text":"Source text."}]}`),
MediaType: "application/json",
Units: append([]source.SourceUnit(nil), req.Source.Units...),
},
func (chunker fakeChunker) Plan(ctx context.Context, req ChunkRequest) (ChunkPlanResult, error) {
return ChunkPlanResult{
Plan: source.ChunkPlan{
SourceDigest: req.Source.Digest,
Ranges: []source.ChunkRange{{
StartUnitID: req.Source.Units[0].ID,
EndUnitID: req.Source.Units[len(req.Source.Units)-1].ID,
}},
},
}, nil
}
@@ -479,9 +454,9 @@ func (chunker *recordingChunker) ReferenceSlots() []ReferenceSlot {
return nil
}
func (chunker *recordingChunker) Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error) {
func (chunker *recordingChunker) Plan(ctx context.Context, req ChunkRequest) (ChunkPlanResult, error) {
chunker.request = req
return fakeChunker{key: chunker.key}.Chunk(ctx, req)
return fakeChunker{key: chunker.key}.Plan(ctx, req)
}
type fakeExtractor struct {