eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move chunks into the canonical source model

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-17 05:32:11 +00:00
parent 40709e4ad8
commit 075888c97f
33 changed files with 548 additions and 325 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
internal/modules/generic/chunk/units/chunker.go
View File

@@ -61,7 +61,7 @@ func (c *Chunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contra
}
step := opts.maxUnits - opts.overlapUnits
chunks := make([]contracts.SourceChunk, 0, (len(req.Source.Units)+step-1)/step)
chunks := make([]source.Chunk, 0, (len(req.Source.Units)+step-1)/step)
for start := 0; start < len(req.Source.Units); start += step {
end := start + opts.maxUnits
if end > len(req.Source.Units) {
@@ -72,15 +72,18 @@ func (c *Chunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contra
if err != nil {
return contracts.ChunkResult{}, err
}
chunks = append(chunks, contracts.SourceChunk{
ID: fmt.Sprintf("chunk-%06d", len(chunks)+1),
SourceID: req.Source.ID,
Index: len(chunks),
StartUnitID: units[0].ID,
EndUnitID: units[len(units)-1].ID,
Content: content,
MediaType: "application/json",
Units: units,
chunks = append(chunks, source.Chunk{
ID: fmt.Sprintf("chunk-%06d", len(chunks)+1),
SourceID: req.Source.ID,
Index: len(chunks),
Ref: source.SourceRef{
SourceID: req.Source.ID,
StartUnitID: units[0].Ref.StartUnitID,
EndUnitID: units[len(units)-1].Ref.EndUnitID,
},
Content: content,
MediaType: "application/json",
Units: units,
Metadata: map[string]any{
"start_unit_id": units[0].ID,
"end_unit_id": units[len(units)-1].ID,