108 lines
3.7 KiB
Go
108 lines
3.7 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
)
|
|
|
|
func validateAndCanonicalizeChunkResult(doc *source.SourceDocument, chunks []contracts.SourceChunk) ([]contracts.SourceChunk, error) {
|
|
if len(chunks) == 0 {
|
|
return nil, fmt.Errorf("chunks must not be empty")
|
|
}
|
|
|
|
sourceUnitIndexes := make(map[int]int, len(doc.Units))
|
|
sourceUnits := make(map[int]source.SourceUnit, len(doc.Units))
|
|
for index, unit := range doc.Units {
|
|
sourceUnitIndexes[unit.ID] = index
|
|
sourceUnits[unit.ID] = unit
|
|
}
|
|
|
|
canonicalChunks := make([]contracts.SourceChunk, 0, len(chunks))
|
|
seenChunkIDs := make(map[string]struct{}, len(chunks))
|
|
for chunkIndex, chunk := range chunks {
|
|
if strings.TrimSpace(chunk.ID) == "" {
|
|
return nil, fmt.Errorf("chunk[%d].id must not be empty", chunkIndex)
|
|
}
|
|
if _, ok := seenChunkIDs[chunk.ID]; ok {
|
|
return nil, fmt.Errorf("chunk id %q is duplicated", chunk.ID)
|
|
}
|
|
seenChunkIDs[chunk.ID] = struct{}{}
|
|
|
|
if chunk.SourceID != doc.ID {
|
|
return nil, fmt.Errorf("chunk %q source_id %q does not match source document id %q", chunk.ID, chunk.SourceID, doc.ID)
|
|
}
|
|
if chunk.Index != chunkIndex {
|
|
return nil, fmt.Errorf("chunk %q index %d does not match returned order %d", chunk.ID, chunk.Index, chunkIndex)
|
|
}
|
|
startIndex, ok := sourceUnitIndexes[chunk.StartUnitID]
|
|
if !ok {
|
|
return nil, fmt.Errorf("chunk %q start_unit_id %d was not found in source document %q", chunk.ID, chunk.StartUnitID, doc.ID)
|
|
}
|
|
endIndex, ok := sourceUnitIndexes[chunk.EndUnitID]
|
|
if !ok {
|
|
return nil, fmt.Errorf("chunk %q end_unit_id %d was not found in source document %q", chunk.ID, chunk.EndUnitID, doc.ID)
|
|
}
|
|
if startIndex > endIndex {
|
|
return nil, fmt.Errorf("chunk %q start_unit_id %d appears after end_unit_id %d", chunk.ID, chunk.StartUnitID, chunk.EndUnitID)
|
|
}
|
|
if len(chunk.Units) == 0 {
|
|
return nil, fmt.Errorf("chunk %q units must not be empty", chunk.ID)
|
|
}
|
|
if len(chunk.Content) == 0 {
|
|
return nil, fmt.Errorf("chunk %q content must not be empty", chunk.ID)
|
|
}
|
|
if strings.TrimSpace(chunk.MediaType) == "" {
|
|
return nil, fmt.Errorf("chunk %q media_type must not be empty", chunk.ID)
|
|
}
|
|
|
|
seenUnitIDs := make(map[int]struct{}, len(chunk.Units))
|
|
previousSourceIndex := -1
|
|
canonicalUnits := make([]source.SourceUnit, 0, len(chunk.Units))
|
|
for unitIndex, unit := range chunk.Units {
|
|
if unit.ID <= 0 {
|
|
return nil, fmt.Errorf("chunk %q unit[%d].id must be positive", chunk.ID, unitIndex)
|
|
}
|
|
if _, ok := seenUnitIDs[unit.ID]; ok {
|
|
return nil, fmt.Errorf("chunk %q repeats source unit %d", chunk.ID, unit.ID)
|
|
}
|
|
seenUnitIDs[unit.ID] = struct{}{}
|
|
|
|
sourceIndex, ok := sourceUnitIndexes[unit.ID]
|
|
if !ok {
|
|
return nil, fmt.Errorf("chunk %q source unit %d was not found in source document %q", chunk.ID, unit.ID, doc.ID)
|
|
}
|
|
if sourceIndex <= previousSourceIndex {
|
|
return nil, fmt.Errorf("chunk %q source units must appear in source document order", chunk.ID)
|
|
}
|
|
previousSourceIndex = sourceIndex
|
|
canonicalUnits = append(canonicalUnits, cloneSourceUnit(sourceUnits[unit.ID]))
|
|
}
|
|
|
|
canonicalChunks = append(canonicalChunks, contracts.SourceChunk{
|
|
ID: chunk.ID,
|
|
SourceID: chunk.SourceID,
|
|
Index: chunk.Index,
|
|
StartUnitID: chunk.StartUnitID,
|
|
EndUnitID: chunk.EndUnitID,
|
|
Content: append([]byte(nil), chunk.Content...),
|
|
MediaType: chunk.MediaType,
|
|
Units: canonicalUnits,
|
|
Metadata: cloneMetadata(chunk.Metadata),
|
|
})
|
|
}
|
|
|
|
return canonicalChunks, nil
|
|
}
|
|
|
|
func cloneSourceUnit(unit source.SourceUnit) source.SourceUnit {
|
|
return source.SourceUnit{
|
|
ID: unit.ID,
|
|
Kind: unit.Kind,
|
|
Text: unit.Text,
|
|
Metadata: cloneMetadata(unit.Metadata),
|
|
}
|
|
}
|