eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Move chunks into the canonical source model

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-17 05:32:11 +00:00
parent 40709e4ad8
commit 075888c97f
33 changed files with 548 additions and 325 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
internal/modules/dnd/chunk/scenes/chunker.go
View File

@@ -135,7 +135,7 @@ func Register(registry *pipeline.ChunkerRegistry) error {
})
}
func chunksFromResponse(doc *source.SourceDocument, response chunkResponse) ([]contracts.SourceChunk, error) {
func chunksFromResponse(doc *source.SourceDocument, response chunkResponse) ([]source.Chunk, error) {
if response.Scenes == nil {
return nil, fmt.Errorf("scenes must be present")
}
@@ -148,7 +148,7 @@ func chunksFromResponse(doc *source.SourceDocument, response chunkResponse) ([]c
unitIndexes[unit.ID] = i
}
chunks := make([]contracts.SourceChunk, 0, len(response.Scenes))
chunks := make([]source.Chunk, 0, len(response.Scenes))
previousEnd := -1
for i, scene := range response.Scenes {
normalized, err := normalizeScene(doc, i, scene)
@@ -186,15 +186,18 @@ func chunksFromResponse(doc *source.SourceDocument, response chunkResponse) ([]c
if err != nil {
return nil, err
}
chunks = append(chunks, contracts.SourceChunk{
ID: fmt.Sprintf("scene-%06d", i+1),
SourceID: doc.ID,
Index: i,
StartUnitID: units[0].ID,
EndUnitID: units[len(units)-1].ID,
Content: content,
MediaType: "application/json",
Units: units,
chunks = append(chunks, source.Chunk{
ID: fmt.Sprintf("scene-%06d", i+1),
SourceID: doc.ID,
Index: i,
Ref: source.SourceRef{
SourceID: doc.ID,
StartUnitID: units[0].Ref.StartUnitID,
EndUnitID: units[len(units)-1].Ref.EndUnitID,
},
Content: content,
MediaType: "application/json",
Units: units,
Metadata: map[string]any{
"scene_title": normalized.ShortTitle,
"primary_mode": normalized.PrimaryMode,

6
internal/modules/dnd/chunk/scenes/chunker_test.go
View File

@@ -179,8 +179,8 @@ func TestChunkReturnsSceneChunksFromStructuredOutput(t *testing.T) {
if first.SourceID != "session-alpha" || first.Index != 0 {
t.Fatalf("first chunk = %#v, want source and index fields", first)
}
if first.StartUnitID != 1 || first.EndUnitID != 2 {
t.Fatalf("first boundaries = %d-%d, want 1-2", first.StartUnitID, first.EndUnitID)
if first.Ref != (source.SourceRef{SourceID: "session-alpha", StartUnitID: 1, EndUnitID: 2}) {
t.Fatalf("first ref = %#v, want session-alpha:1-2", first.Ref)
}
if first.MediaType != "application/json" || len(first.Content) == 0 {
t.Fatalf("first payload = media type %q length %d, want JSON content", first.MediaType, len(first.Content))
@@ -574,7 +574,7 @@ func scene(startUnitID int, endUnitID int) sceneResponse {
}
}
func chunkIDs(chunks []contracts.SourceChunk) []string {
func chunkIDs(chunks []source.Chunk) []string {
ids := make([]string, 0, len(chunks))
for _, chunk := range chunks {
ids = append(ids, chunk.ID)