83 lines
2.8 KiB
Go
83 lines
2.8 KiB
Go
package pipeline
|
|
|
|
import (
|
|
"fmt"
|
|
"strings"
|
|
|
|
"gitea.maximumdirect.net/eric/notarius/internal/core/source"
|
|
"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
|
|
)
|
|
|
|
func validateAndCanonicalizeChunkResult(doc *source.SourceDocument, chunks []contracts.SourceChunk) ([]contracts.SourceChunk, error) {
|
|
sourceUnitIndexes := make(map[string]int, len(doc.Units))
|
|
sourceUnits := make(map[string]source.SourceUnit, len(doc.Units))
|
|
for index, unit := range doc.Units {
|
|
sourceUnitIndexes[unit.ID] = index
|
|
sourceUnits[unit.ID] = unit
|
|
}
|
|
|
|
canonicalChunks := make([]contracts.SourceChunk, 0, len(chunks))
|
|
seenChunkIDs := make(map[string]struct{}, len(chunks))
|
|
for chunkIndex, chunk := range chunks {
|
|
if strings.TrimSpace(chunk.ID) == "" {
|
|
return nil, fmt.Errorf("chunk[%d].id must not be empty", chunkIndex)
|
|
}
|
|
if _, ok := seenChunkIDs[chunk.ID]; ok {
|
|
return nil, fmt.Errorf("chunk id %q is duplicated", chunk.ID)
|
|
}
|
|
seenChunkIDs[chunk.ID] = struct{}{}
|
|
|
|
if chunk.SourceID != doc.ID {
|
|
return nil, fmt.Errorf("chunk %q source_id %q does not match source document id %q", chunk.ID, chunk.SourceID, doc.ID)
|
|
}
|
|
if chunk.Index != chunkIndex {
|
|
return nil, fmt.Errorf("chunk %q index %d does not match returned order %d", chunk.ID, chunk.Index, chunkIndex)
|
|
}
|
|
if len(chunk.Units) == 0 {
|
|
return nil, fmt.Errorf("chunk %q units must not be empty", chunk.ID)
|
|
}
|
|
|
|
seenUnitIDs := make(map[string]struct{}, len(chunk.Units))
|
|
previousSourceIndex := -1
|
|
canonicalUnits := make([]source.SourceUnit, 0, len(chunk.Units))
|
|
for unitIndex, unit := range chunk.Units {
|
|
if strings.TrimSpace(unit.ID) == "" {
|
|
return nil, fmt.Errorf("chunk %q unit[%d].id must not be empty", chunk.ID, unitIndex)
|
|
}
|
|
if _, ok := seenUnitIDs[unit.ID]; ok {
|
|
return nil, fmt.Errorf("chunk %q repeats source unit %q", chunk.ID, unit.ID)
|
|
}
|
|
seenUnitIDs[unit.ID] = struct{}{}
|
|
|
|
sourceIndex, ok := sourceUnitIndexes[unit.ID]
|
|
if !ok {
|
|
return nil, fmt.Errorf("chunk %q source unit %q was not found in source document %q", chunk.ID, unit.ID, doc.ID)
|
|
}
|
|
if sourceIndex <= previousSourceIndex {
|
|
return nil, fmt.Errorf("chunk %q source units must appear in source document order", chunk.ID)
|
|
}
|
|
previousSourceIndex = sourceIndex
|
|
canonicalUnits = append(canonicalUnits, cloneSourceUnit(sourceUnits[unit.ID]))
|
|
}
|
|
|
|
canonicalChunks = append(canonicalChunks, contracts.SourceChunk{
|
|
ID: chunk.ID,
|
|
SourceID: chunk.SourceID,
|
|
Index: chunk.Index,
|
|
Units: canonicalUnits,
|
|
Metadata: cloneMetadata(chunk.Metadata),
|
|
})
|
|
}
|
|
|
|
return canonicalChunks, nil
|
|
}
|
|
|
|
func cloneSourceUnit(unit source.SourceUnit) source.SourceUnit {
|
|
return source.SourceUnit{
|
|
ID: unit.ID,
|
|
Kind: unit.Kind,
|
|
Text: unit.Text,
|
|
Metadata: cloneMetadata(unit.Metadata),
|
|
}
|
|
}
|