Implement integer source units and chunk payloads

internal/modules/chunk/dnd/scenes/chunker.go

@@ -2,6 +2,7 @@ package scenes
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"strings"
 
@@ -142,7 +143,7 @@ func chunksFromResponse(doc *source.SourceDocument, response chunkResponse) ([]c
 		return nil, fmt.Errorf("scenes must not be empty")
 	}
 
-	unitIndexes := make(map[string]int, len(doc.Units))
+	unitIndexes := make(map[int]int, len(doc.Units))
 	for i, unit := range doc.Units {
 		unitIndexes[unit.ID] = i
 	}
@@ -157,18 +158,18 @@ func chunksFromResponse(doc *source.SourceDocument, response chunkResponse) ([]c
 
 		startIndex, ok := unitIndexes[normalized.StartUnitID]
 		if !ok {
-			return nil, fmt.Errorf("scene[%d] start_unit_id %q was not found", i, normalized.StartUnitID)
+			return nil, fmt.Errorf("scene[%d] start_unit_id %d was not found", i, normalized.StartUnitID)
 		}
 		endIndex, ok := unitIndexes[normalized.EndUnitID]
 		if !ok {
-			return nil, fmt.Errorf("scene[%d] end_unit_id %q was not found", i, normalized.EndUnitID)
+			return nil, fmt.Errorf("scene[%d] end_unit_id %d was not found", i, normalized.EndUnitID)
 		}
 		if startIndex > endIndex {
-			return nil, fmt.Errorf("scene[%d] start_unit_id %q appears after end_unit_id %q", i, normalized.StartUnitID, normalized.EndUnitID)
+			return nil, fmt.Errorf("scene[%d] start_unit_id %d appears after end_unit_id %d", i, normalized.StartUnitID, normalized.EndUnitID)
 		}
 
 		if i == 0 && startIndex != 0 {
-			return nil, fmt.Errorf("first scene must start at first source unit %q", doc.Units[0].ID)
+			return nil, fmt.Errorf("first scene must start at first source unit %d", doc.Units[0].ID)
 		}
 		if i > 0 {
 			if startIndex <= previousEnd {
@@ -181,11 +182,19 @@ func chunksFromResponse(doc *source.SourceDocument, response chunkResponse) ([]c
 		previousEnd = endIndex
 
 		units := cloneUnits(doc.Units[startIndex : endIndex+1])
+		content, err := chunkContent(units)
+		if err != nil {
+			return nil, err
+		}
 		chunks = append(chunks, contracts.SourceChunk{
-			ID:       fmt.Sprintf("scene-%06d", i+1),
-			SourceID: doc.ID,
-			Index:    i,
-			Units:    units,
+			ID:          fmt.Sprintf("scene-%06d", i+1),
+			SourceID:    doc.ID,
+			Index:       i,
+			StartUnitID: units[0].ID,
+			EndUnitID:   units[len(units)-1].ID,
+			Content:     content,
+			MediaType:   "application/json",
+			Units:       units,
 			Metadata: map[string]any{
 				"scene_title":         normalized.ShortTitle,
 				"primary_mode":        normalized.PrimaryMode,
@@ -201,11 +210,23 @@ func chunksFromResponse(doc *source.SourceDocument, response chunkResponse) ([]c
 	}
 
 	if previousEnd != len(doc.Units)-1 {
-		return nil, fmt.Errorf("final scene must end at final source unit %q", doc.Units[len(doc.Units)-1].ID)
+		return nil, fmt.Errorf("final scene must end at final source unit %d", doc.Units[len(doc.Units)-1].ID)
 	}
 	return chunks, nil
 }
 
+func chunkContent(units []source.SourceUnit) ([]byte, error) {
+	content, err := json.Marshal(struct {
+		Units []source.SourceUnit `json:"units"`
+	}{
+		Units: units,
+	})
+	if err != nil {
+		return nil, fmt.Errorf("encode chunk content: %w", err)
+	}
+	return content, nil
+}
+
 func normalizeScene(doc *source.SourceDocument, index int, scene sceneResponse) (normalizedScene, error) {
 	startUnitID, err := dnd.ResolveUnitID(doc, "start_unit_id", scene.StartUnitID)
 	if err != nil {
@@ -226,9 +247,16 @@ func normalizeScene(doc *source.SourceDocument, index int, scene sceneResponse)
 		BoundaryConfidence: strings.TrimSpace(scene.BoundaryConfidence),
 	}
 
+	requiredInts := map[string]int{
+		"start_unit_id": out.StartUnitID,
+		"end_unit_id":   out.EndUnitID,
+	}
+	for field, value := range requiredInts {
+		if value <= 0 {
+			return normalizedScene{}, fmt.Errorf("scene[%d] %s must be positive", index, field)
+		}
+	}
 	required := map[string]string{
-		"start_unit_id":       out.StartUnitID,
-		"end_unit_id":         out.EndUnitID,
 		"short_title":         out.ShortTitle,
 		"primary_mode":        out.PrimaryMode,
 		"summary":             out.Summary,