Add canonical chunk plan source model
This commit is contained in:
@@ -48,6 +48,14 @@ func validateAndCanonicalizeChunkResult(doc *source.SourceDocument, chunks []sou
|
||||
if err := source.ValidateRef(doc, chunk.Ref); err != nil {
|
||||
return nil, fmt.Errorf("chunk %q ref: %w", chunk.ID, err)
|
||||
}
|
||||
annotations, err := source.CanonicalizeChunkAnnotations(chunk.Annotations)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("chunk %q annotations: %w", chunk.ID, err)
|
||||
}
|
||||
planAnnotations, err := source.CanonicalizeChunkAnnotations(chunk.PlanAnnotations)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("chunk %q plan_annotations: %w", chunk.ID, err)
|
||||
}
|
||||
|
||||
seenUnitIDs := make(map[int]struct{}, len(chunk.Units))
|
||||
previousSourceIndex := -1
|
||||
@@ -84,14 +92,16 @@ func validateAndCanonicalizeChunkResult(doc *source.SourceDocument, chunks []sou
|
||||
}
|
||||
|
||||
canonicalChunks = append(canonicalChunks, source.Chunk{
|
||||
ID: chunk.ID,
|
||||
SourceID: chunk.SourceID,
|
||||
Index: chunk.Index,
|
||||
Ref: expectedRef,
|
||||
Content: append([]byte(nil), chunk.Content...),
|
||||
MediaType: chunk.MediaType,
|
||||
Units: canonicalUnits,
|
||||
Metadata: cloneMetadata(chunk.Metadata),
|
||||
ID: chunk.ID,
|
||||
SourceID: chunk.SourceID,
|
||||
Index: chunk.Index,
|
||||
Ref: expectedRef,
|
||||
Content: append([]byte(nil), chunk.Content...),
|
||||
MediaType: chunk.MediaType,
|
||||
Units: canonicalUnits,
|
||||
Metadata: cloneMetadata(chunk.Metadata),
|
||||
Annotations: annotations,
|
||||
PlanAnnotations: planAnnotations,
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user