eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Generate and materialize canonical chunk plans

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-17 23:57:59 +00:00
parent 3bfac14397
commit 7844c0a93f
23 changed files with 396 additions and 540 deletions
Download Patch File Download Diff File Expand all files Collapse all files

82
internal/modules/generic/chunk/units/chunker.go
View File

@@ -42,77 +42,46 @@ func (c *Chunker) ReferenceSlots() []contracts.ReferenceSlot {
return nil
}
func (c *Chunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
func (c *Chunker) Plan(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
if c == nil {
return contracts.ChunkResult{}, chunkerErrorf("chunker must not be nil")
return contracts.ChunkPlanResult{}, chunkerErrorf("chunker must not be nil")
}
if c.options.MaxUnits <= 0 || c.options.OverlapUnits < 0 || c.options.OverlapUnits >= c.options.MaxUnits {
return contracts.ChunkResult{}, chunkerErrorf("chunker options must be initialized by construction")
return contracts.ChunkPlanResult{}, chunkerErrorf("chunker options must be initialized by construction")
}
if ctx == nil {
return contracts.ChunkResult{}, chunkerErrorf("context must not be nil")
return contracts.ChunkPlanResult{}, chunkerErrorf("context must not be nil")
}
if err := ctx.Err(); err != nil {
return contracts.ChunkResult{}, chunkerErrorf("context error before chunking: %w", err)
return contracts.ChunkPlanResult{}, chunkerErrorf("context error before chunking: %w", err)
}
if req.Source == nil {
return contracts.ChunkResult{}, chunkerErrorf("source must not be nil")
return contracts.ChunkPlanResult{}, chunkerErrorf("source must not be nil")
}
if len(req.Source.Units) == 0 {
return contracts.ChunkResult{}, chunkerErrorf("source units must not be empty")
return contracts.ChunkPlanResult{}, chunkerErrorf("source units must not be empty")
}
if err := source.ValidateDocument(req.Source); err != nil {
return contracts.ChunkResult{}, chunkerErrorf("validate source document: %w", err)
return contracts.ChunkPlanResult{}, chunkerErrorf("validate source document: %w", err)
}
step := c.options.MaxUnits - c.options.OverlapUnits
chunks := make([]source.Chunk, 0, (len(req.Source.Units)+step-1)/step)
ranges := make([]source.ChunkRange, 0, (len(req.Source.Units)+step-1)/step)
for start := 0; start < len(req.Source.Units); start += step {
end := start + c.options.MaxUnits
if end > len(req.Source.Units) {
end = len(req.Source.Units)
}
units := cloneUnits(req.Source.Units[start:end])
content, err := chunkContent(units)
if err != nil {
return contracts.ChunkResult{}, err
}
chunks = append(chunks, source.Chunk{
ID: fmt.Sprintf("chunk-%06d", len(chunks)+1),
SourceID: req.Source.ID,
Index: len(chunks),
Ref: source.SourceRef{
SourceID: req.Source.ID,
StartUnitID: units[0].Ref.StartUnitID,
EndUnitID: units[len(units)-1].Ref.EndUnitID,
},
Content: content,
MediaType: "application/json",
Units: units,
Metadata: map[string]any{
"start_unit_id": units[0].ID,
"end_unit_id": units[len(units)-1].ID,
"unit_count": len(units),
},
ranges = append(ranges, source.ChunkRange{
StartUnitID: req.Source.Units[start].ID,
EndUnitID: req.Source.Units[end-1].ID,
})
if end == len(req.Source.Units) {
break
}
}
return contracts.ChunkResult{Chunks: chunks}, nil
}
func chunkContent(units []source.SourceUnit) ([]byte, error) {
content, err := json.Marshal(struct {
Units []source.SourceUnit `json:"units"`
}{
Units: units,
})
if err != nil {
return nil, chunkerErrorf("encode chunk content: %w", err)
}
return content, nil
return contracts.ChunkPlanResult{Plan: source.ChunkPlan{SourceDigest: req.Source.Digest, Ranges: ranges}}, nil
}
func ModuleSpec() pipeline.ModuleSpec {
@@ -251,31 +220,6 @@ func minInt() int64 {
return -maxInt() - 1
}
func cloneUnits(units []source.SourceUnit) []source.SourceUnit {
out := make([]source.SourceUnit, 0, len(units))
for _, unit := range units {
out = append(out, source.SourceUnit{
ID: unit.ID,
Kind: unit.Kind,
Text: unit.Text,
Ref: unit.Ref,
Metadata: cloneMetadata(unit.Metadata),
})
}
return out
}
func cloneMetadata(metadata map[string]any) map[string]any {
if len(metadata) == 0 {
return nil
}
out := make(map[string]any, len(metadata))
for key, value := range metadata {
out[key] = value
}
return out
}
func chunkerErrorf(format string, args ...any) error {
return fmt.Errorf("generic chunker: "+format, args...)
}