Add canonical chunk plan source model

This commit is contained in:
2026-07-17 23:47:22 +00:00
parent 1c13e1d64a
commit 3bfac14397
16 changed files with 755 additions and 73 deletions

View File

@@ -48,6 +48,14 @@ func validateAndCanonicalizeChunkResult(doc *source.SourceDocument, chunks []sou
if err := source.ValidateRef(doc, chunk.Ref); err != nil {
return nil, fmt.Errorf("chunk %q ref: %w", chunk.ID, err)
}
annotations, err := source.CanonicalizeChunkAnnotations(chunk.Annotations)
if err != nil {
return nil, fmt.Errorf("chunk %q annotations: %w", chunk.ID, err)
}
planAnnotations, err := source.CanonicalizeChunkAnnotations(chunk.PlanAnnotations)
if err != nil {
return nil, fmt.Errorf("chunk %q plan_annotations: %w", chunk.ID, err)
}
seenUnitIDs := make(map[int]struct{}, len(chunk.Units))
previousSourceIndex := -1
@@ -84,14 +92,16 @@ func validateAndCanonicalizeChunkResult(doc *source.SourceDocument, chunks []sou
}
canonicalChunks = append(canonicalChunks, source.Chunk{
ID: chunk.ID,
SourceID: chunk.SourceID,
Index: chunk.Index,
Ref: expectedRef,
Content: append([]byte(nil), chunk.Content...),
MediaType: chunk.MediaType,
Units: canonicalUnits,
Metadata: cloneMetadata(chunk.Metadata),
ID: chunk.ID,
SourceID: chunk.SourceID,
Index: chunk.Index,
Ref: expectedRef,
Content: append([]byte(nil), chunk.Content...),
MediaType: chunk.MediaType,
Units: canonicalUnits,
Metadata: cloneMetadata(chunk.Metadata),
Annotations: annotations,
PlanAnnotations: planAnnotations,
})
}