Canonicalize chunk units before extraction

internal/framework/pipeline/chunk_validation.go

@@ -8,53 +8,75 @@ import (
 	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
 )
 
-func validateChunkResult(doc *source.SourceDocument, chunks []contracts.SourceChunk) error {
+func validateAndCanonicalizeChunkResult(doc *source.SourceDocument, chunks []contracts.SourceChunk) ([]contracts.SourceChunk, error) {
 	sourceUnitIndexes := make(map[string]int, len(doc.Units))
+	sourceUnits := make(map[string]source.SourceUnit, len(doc.Units))
 	for index, unit := range doc.Units {
 		sourceUnitIndexes[unit.ID] = index
+		sourceUnits[unit.ID] = unit
 	}
 
+	canonicalChunks := make([]contracts.SourceChunk, 0, len(chunks))
 	seenChunkIDs := make(map[string]struct{}, len(chunks))
 	for chunkIndex, chunk := range chunks {
 		if strings.TrimSpace(chunk.ID) == "" {
-			return fmt.Errorf("chunk[%d].id must not be empty", chunkIndex)
+			return nil, fmt.Errorf("chunk[%d].id must not be empty", chunkIndex)
 		}
 		if _, ok := seenChunkIDs[chunk.ID]; ok {
-			return fmt.Errorf("chunk id %q is duplicated", chunk.ID)
+			return nil, fmt.Errorf("chunk id %q is duplicated", chunk.ID)
 		}
 		seenChunkIDs[chunk.ID] = struct{}{}
 
 		if chunk.SourceID != doc.ID {
-			return fmt.Errorf("chunk %q source_id %q does not match source document id %q", chunk.ID, chunk.SourceID, doc.ID)
+			return nil, fmt.Errorf("chunk %q source_id %q does not match source document id %q", chunk.ID, chunk.SourceID, doc.ID)
 		}
 		if chunk.Index != chunkIndex {
-			return fmt.Errorf("chunk %q index %d does not match returned order %d", chunk.ID, chunk.Index, chunkIndex)
+			return nil, fmt.Errorf("chunk %q index %d does not match returned order %d", chunk.ID, chunk.Index, chunkIndex)
 		}
 		if len(chunk.Units) == 0 {
-			return fmt.Errorf("chunk %q units must not be empty", chunk.ID)
+			return nil, fmt.Errorf("chunk %q units must not be empty", chunk.ID)
 		}
 
 		seenUnitIDs := make(map[string]struct{}, len(chunk.Units))
 		previousSourceIndex := -1
+		canonicalUnits := make([]source.SourceUnit, 0, len(chunk.Units))
 		for unitIndex, unit := range chunk.Units {
 			if strings.TrimSpace(unit.ID) == "" {
-				return fmt.Errorf("chunk %q unit[%d].id must not be empty", chunk.ID, unitIndex)
+				return nil, fmt.Errorf("chunk %q unit[%d].id must not be empty", chunk.ID, unitIndex)
 			}
 			if _, ok := seenUnitIDs[unit.ID]; ok {
-				return fmt.Errorf("chunk %q repeats source unit %q", chunk.ID, unit.ID)
+				return nil, fmt.Errorf("chunk %q repeats source unit %q", chunk.ID, unit.ID)
 			}
 			seenUnitIDs[unit.ID] = struct{}{}
 
 			sourceIndex, ok := sourceUnitIndexes[unit.ID]
 			if !ok {
-				return fmt.Errorf("chunk %q source unit %q was not found in source document %q", chunk.ID, unit.ID, doc.ID)
+				return nil, fmt.Errorf("chunk %q source unit %q was not found in source document %q", chunk.ID, unit.ID, doc.ID)
 			}
 			if sourceIndex <= previousSourceIndex {
-				return fmt.Errorf("chunk %q source units must appear in source document order", chunk.ID)
+				return nil, fmt.Errorf("chunk %q source units must appear in source document order", chunk.ID)
 			}
 			previousSourceIndex = sourceIndex
+			canonicalUnits = append(canonicalUnits, cloneSourceUnit(sourceUnits[unit.ID]))
 		}
+
+		canonicalChunks = append(canonicalChunks, contracts.SourceChunk{
+			ID:       chunk.ID,
+			SourceID: chunk.SourceID,
+			Index:    chunk.Index,
+			Units:    canonicalUnits,
+			Metadata: cloneMetadata(chunk.Metadata),
+		})
 	}
 
-	return nil
+	return canonicalChunks, nil
+}
+
+func cloneSourceUnit(unit source.SourceUnit) source.SourceUnit {
+	return source.SourceUnit{
+		ID:       unit.ID,
+		Kind:     unit.Kind,
+		Text:     unit.Text,
+		Metadata: cloneMetadata(unit.Metadata),
+	}
 }