package pipeline import ( "fmt" "strings" "gitea.maximumdirect.net/eric/notarius/internal/core/source" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" ) func validateAndCanonicalizeChunkResult(doc *source.SourceDocument, chunks []contracts.SourceChunk) ([]contracts.SourceChunk, error) { if len(chunks) == 0 { return nil, fmt.Errorf("chunks must not be empty") } sourceUnitIndexes := make(map[int]int, len(doc.Units)) sourceUnits := make(map[int]source.SourceUnit, len(doc.Units)) for index, unit := range doc.Units { sourceUnitIndexes[unit.ID] = index sourceUnits[unit.ID] = unit } canonicalChunks := make([]contracts.SourceChunk, 0, len(chunks)) seenChunkIDs := make(map[string]struct{}, len(chunks)) for chunkIndex, chunk := range chunks { if strings.TrimSpace(chunk.ID) == "" { return nil, fmt.Errorf("chunk[%d].id must not be empty", chunkIndex) } if _, ok := seenChunkIDs[chunk.ID]; ok { return nil, fmt.Errorf("chunk id %q is duplicated", chunk.ID) } seenChunkIDs[chunk.ID] = struct{}{} if chunk.SourceID != doc.ID { return nil, fmt.Errorf("chunk %q source_id %q does not match source document id %q", chunk.ID, chunk.SourceID, doc.ID) } if chunk.Index != chunkIndex { return nil, fmt.Errorf("chunk %q index %d does not match returned order %d", chunk.ID, chunk.Index, chunkIndex) } startIndex, ok := sourceUnitIndexes[chunk.StartUnitID] if !ok { return nil, fmt.Errorf("chunk %q start_unit_id %d was not found in source document %q", chunk.ID, chunk.StartUnitID, doc.ID) } endIndex, ok := sourceUnitIndexes[chunk.EndUnitID] if !ok { return nil, fmt.Errorf("chunk %q end_unit_id %d was not found in source document %q", chunk.ID, chunk.EndUnitID, doc.ID) } if startIndex > endIndex { return nil, fmt.Errorf("chunk %q start_unit_id %d appears after end_unit_id %d", chunk.ID, chunk.StartUnitID, chunk.EndUnitID) } if len(chunk.Units) == 0 { return nil, fmt.Errorf("chunk %q units must not be empty", chunk.ID) } if len(chunk.Content) == 0 { return nil, fmt.Errorf("chunk %q content must not be empty", chunk.ID) } if strings.TrimSpace(chunk.MediaType) == "" { return nil, fmt.Errorf("chunk %q media_type must not be empty", chunk.ID) } seenUnitIDs := make(map[int]struct{}, len(chunk.Units)) previousSourceIndex := -1 canonicalUnits := make([]source.SourceUnit, 0, len(chunk.Units)) for unitIndex, unit := range chunk.Units { if unit.ID <= 0 { return nil, fmt.Errorf("chunk %q unit[%d].id must be positive", chunk.ID, unitIndex) } if _, ok := seenUnitIDs[unit.ID]; ok { return nil, fmt.Errorf("chunk %q repeats source unit %d", chunk.ID, unit.ID) } seenUnitIDs[unit.ID] = struct{}{} sourceIndex, ok := sourceUnitIndexes[unit.ID] if !ok { return nil, fmt.Errorf("chunk %q source unit %d was not found in source document %q", chunk.ID, unit.ID, doc.ID) } if sourceIndex <= previousSourceIndex { return nil, fmt.Errorf("chunk %q source units must appear in source document order", chunk.ID) } previousSourceIndex = sourceIndex canonicalUnits = append(canonicalUnits, cloneSourceUnit(sourceUnits[unit.ID])) } canonicalChunks = append(canonicalChunks, contracts.SourceChunk{ ID: chunk.ID, SourceID: chunk.SourceID, Index: chunk.Index, StartUnitID: chunk.StartUnitID, EndUnitID: chunk.EndUnitID, Content: append([]byte(nil), chunk.Content...), MediaType: chunk.MediaType, Units: canonicalUnits, Metadata: cloneMetadata(chunk.Metadata), }) } return canonicalChunks, nil } func cloneSourceUnit(unit source.SourceUnit) source.SourceUnit { return source.SourceUnit{ ID: unit.ID, Kind: unit.Kind, Text: unit.Text, Metadata: cloneMetadata(unit.Metadata), } }