Implement integer source units and chunk payloads

internal/framework/pipeline/chunk_validation.go

@@ -9,8 +9,12 @@ import (
 )
 
 func validateAndCanonicalizeChunkResult(doc *source.SourceDocument, chunks []contracts.SourceChunk) ([]contracts.SourceChunk, error) {
-	sourceUnitIndexes := make(map[string]int, len(doc.Units))
-	sourceUnits := make(map[string]source.SourceUnit, len(doc.Units))
+	if len(chunks) == 0 {
+		return nil, fmt.Errorf("chunks must not be empty")
+	}
+
+	sourceUnitIndexes := make(map[int]int, len(doc.Units))
+	sourceUnits := make(map[int]source.SourceUnit, len(doc.Units))
 	for index, unit := range doc.Units {
 		sourceUnitIndexes[unit.ID] = index
 		sourceUnits[unit.ID] = unit
@@ -33,25 +37,42 @@ func validateAndCanonicalizeChunkResult(doc *source.SourceDocument, chunks []con
 		if chunk.Index != chunkIndex {
 			return nil, fmt.Errorf("chunk %q index %d does not match returned order %d", chunk.ID, chunk.Index, chunkIndex)
 		}
+		startIndex, ok := sourceUnitIndexes[chunk.StartUnitID]
+		if !ok {
+			return nil, fmt.Errorf("chunk %q start_unit_id %d was not found in source document %q", chunk.ID, chunk.StartUnitID, doc.ID)
+		}
+		endIndex, ok := sourceUnitIndexes[chunk.EndUnitID]
+		if !ok {
+			return nil, fmt.Errorf("chunk %q end_unit_id %d was not found in source document %q", chunk.ID, chunk.EndUnitID, doc.ID)
+		}
+		if startIndex > endIndex {
+			return nil, fmt.Errorf("chunk %q start_unit_id %d appears after end_unit_id %d", chunk.ID, chunk.StartUnitID, chunk.EndUnitID)
+		}
 		if len(chunk.Units) == 0 {
 			return nil, fmt.Errorf("chunk %q units must not be empty", chunk.ID)
 		}
+		if len(chunk.Content) == 0 {
+			return nil, fmt.Errorf("chunk %q content must not be empty", chunk.ID)
+		}
+		if strings.TrimSpace(chunk.MediaType) == "" {
+			return nil, fmt.Errorf("chunk %q media_type must not be empty", chunk.ID)
+		}
 
-		seenUnitIDs := make(map[string]struct{}, len(chunk.Units))
+		seenUnitIDs := make(map[int]struct{}, len(chunk.Units))
 		previousSourceIndex := -1
 		canonicalUnits := make([]source.SourceUnit, 0, len(chunk.Units))
 		for unitIndex, unit := range chunk.Units {
-			if strings.TrimSpace(unit.ID) == "" {
-				return nil, fmt.Errorf("chunk %q unit[%d].id must not be empty", chunk.ID, unitIndex)
+			if unit.ID <= 0 {
+				return nil, fmt.Errorf("chunk %q unit[%d].id must be positive", chunk.ID, unitIndex)
 			}
 			if _, ok := seenUnitIDs[unit.ID]; ok {
-				return nil, fmt.Errorf("chunk %q repeats source unit %q", chunk.ID, unit.ID)
+				return nil, fmt.Errorf("chunk %q repeats source unit %d", chunk.ID, unit.ID)
 			}
 			seenUnitIDs[unit.ID] = struct{}{}
 
 			sourceIndex, ok := sourceUnitIndexes[unit.ID]
 			if !ok {
-				return nil, fmt.Errorf("chunk %q source unit %q was not found in source document %q", chunk.ID, unit.ID, doc.ID)
+				return nil, fmt.Errorf("chunk %q source unit %d was not found in source document %q", chunk.ID, unit.ID, doc.ID)
 			}
 			if sourceIndex <= previousSourceIndex {
 				return nil, fmt.Errorf("chunk %q source units must appear in source document order", chunk.ID)
@@ -61,11 +82,15 @@ func validateAndCanonicalizeChunkResult(doc *source.SourceDocument, chunks []con
 		}
 
 		canonicalChunks = append(canonicalChunks, contracts.SourceChunk{
-			ID:       chunk.ID,
-			SourceID: chunk.SourceID,
-			Index:    chunk.Index,
-			Units:    canonicalUnits,
-			Metadata: cloneMetadata(chunk.Metadata),
+			ID:          chunk.ID,
+			SourceID:    chunk.SourceID,
+			Index:       chunk.Index,
+			StartUnitID: chunk.StartUnitID,
+			EndUnitID:   chunk.EndUnitID,
+			Content:     append([]byte(nil), chunk.Content...),
+			MediaType:   chunk.MediaType,
+			Units:       canonicalUnits,
+			Metadata:    cloneMetadata(chunk.Metadata),
 		})
 	}