120 lines
4.2 KiB
Go
120 lines
4.2 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
)
|
|
|
|
func validateAndCanonicalizeChunkResult(doc *source.SourceDocument, chunks []source.Chunk) ([]source.Chunk, error) {
|
|
if len(chunks) == 0 {
|
|
return nil, fmt.Errorf("chunks must not be empty")
|
|
}
|
|
|
|
sourceUnitIndexes := make(map[int]int, len(doc.Units))
|
|
sourceUnits := make(map[int]source.SourceUnit, len(doc.Units))
|
|
for index, unit := range doc.Units {
|
|
sourceUnitIndexes[unit.ID] = index
|
|
sourceUnits[unit.ID] = unit
|
|
}
|
|
|
|
canonicalChunks := make([]source.Chunk, 0, len(chunks))
|
|
seenChunkIDs := make(map[string]struct{}, len(chunks))
|
|
for chunkIndex, chunk := range chunks {
|
|
if strings.TrimSpace(chunk.ID) == "" {
|
|
return nil, fmt.Errorf("chunk[%d].id must not be empty", chunkIndex)
|
|
}
|
|
if _, ok := seenChunkIDs[chunk.ID]; ok {
|
|
return nil, fmt.Errorf("chunk id %q is duplicated", chunk.ID)
|
|
}
|
|
seenChunkIDs[chunk.ID] = struct{}{}
|
|
|
|
if chunk.SourceID != doc.ID {
|
|
return nil, fmt.Errorf("chunk %q source_id %q does not match source document id %q", chunk.ID, chunk.SourceID, doc.ID)
|
|
}
|
|
if chunk.Index != chunkIndex {
|
|
return nil, fmt.Errorf("chunk %q index %d does not match returned order %d", chunk.ID, chunk.Index, chunkIndex)
|
|
}
|
|
if len(chunk.Units) == 0 {
|
|
return nil, fmt.Errorf("chunk %q units must not be empty", chunk.ID)
|
|
}
|
|
if len(chunk.Content) == 0 {
|
|
return nil, fmt.Errorf("chunk %q content must not be empty", chunk.ID)
|
|
}
|
|
if strings.TrimSpace(chunk.MediaType) == "" {
|
|
return nil, fmt.Errorf("chunk %q media_type must not be empty", chunk.ID)
|
|
}
|
|
if err := source.ValidateRef(doc, chunk.Ref); err != nil {
|
|
return nil, fmt.Errorf("chunk %q ref: %w", chunk.ID, err)
|
|
}
|
|
annotations, err := source.CanonicalizeChunkAnnotations(chunk.Annotations)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("chunk %q annotations: %w", chunk.ID, err)
|
|
}
|
|
planAnnotations, err := source.CanonicalizeChunkAnnotations(chunk.PlanAnnotations)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("chunk %q plan_annotations: %w", chunk.ID, err)
|
|
}
|
|
|
|
seenUnitIDs := make(map[int]struct{}, len(chunk.Units))
|
|
previousSourceIndex := -1
|
|
canonicalUnits := make([]source.SourceUnit, 0, len(chunk.Units))
|
|
for unitIndex, unit := range chunk.Units {
|
|
if unit.ID <= 0 {
|
|
return nil, fmt.Errorf("chunk %q unit[%d].id must be positive", chunk.ID, unitIndex)
|
|
}
|
|
if _, ok := seenUnitIDs[unit.ID]; ok {
|
|
return nil, fmt.Errorf("chunk %q repeats source unit %d", chunk.ID, unit.ID)
|
|
}
|
|
seenUnitIDs[unit.ID] = struct{}{}
|
|
|
|
sourceIndex, ok := sourceUnitIndexes[unit.ID]
|
|
if !ok {
|
|
return nil, fmt.Errorf("chunk %q source unit %d was not found in source document %q", chunk.ID, unit.ID, doc.ID)
|
|
}
|
|
if previousSourceIndex >= 0 && sourceIndex != previousSourceIndex+1 {
|
|
return nil, fmt.Errorf("chunk %q source units must form a contiguous range in source document order", chunk.ID)
|
|
}
|
|
if unit.Ref != sourceUnits[unit.ID].Ref {
|
|
return nil, fmt.Errorf("chunk %q source unit %d ref does not match source document", chunk.ID, unit.ID)
|
|
}
|
|
previousSourceIndex = sourceIndex
|
|
canonicalUnits = append(canonicalUnits, cloneSourceUnit(sourceUnits[unit.ID]))
|
|
}
|
|
expectedRef := source.SourceRef{
|
|
SourceID: doc.ID,
|
|
StartUnitID: canonicalUnits[0].Ref.StartUnitID,
|
|
EndUnitID: canonicalUnits[len(canonicalUnits)-1].Ref.EndUnitID,
|
|
}
|
|
if chunk.Ref != expectedRef {
|
|
return nil, fmt.Errorf("chunk %q ref %#v does not match unit span %#v", chunk.ID, chunk.Ref, expectedRef)
|
|
}
|
|
|
|
canonicalChunks = append(canonicalChunks, source.Chunk{
|
|
ID: chunk.ID,
|
|
SourceID: chunk.SourceID,
|
|
Index: chunk.Index,
|
|
Ref: expectedRef,
|
|
Content: append([]byte(nil), chunk.Content...),
|
|
MediaType: chunk.MediaType,
|
|
Units: canonicalUnits,
|
|
Metadata: cloneMetadata(chunk.Metadata),
|
|
Annotations: annotations,
|
|
PlanAnnotations: planAnnotations,
|
|
})
|
|
}
|
|
|
|
return canonicalChunks, nil
|
|
}
|
|
|
|
func cloneSourceUnit(unit source.SourceUnit) source.SourceUnit {
|
|
return source.SourceUnit{
|
|
ID: unit.ID,
|
|
Kind: unit.Kind,
|
|
Text: unit.Text,
|
|
Ref: unit.Ref,
|
|
Metadata: cloneMetadata(unit.Metadata),
|
|
}
|
|
}
|