Move chunks into the canonical source model

internal/core/source/digest.go

@@ -33,3 +33,33 @@ func DigestDocument(doc *SourceDocument) (string, error) {
 	sum := sha256.Sum256(encoded)
 	return "sha256:" + hex.EncodeToString(sum[:]), nil
 }
+
+// DigestChunk returns a deterministic digest of a chunk, including its source
+// provenance, content, units, and metadata.
+func DigestChunk(chunk Chunk) (string, error) {
+	payload := struct {
+		ID        string         `json:"id"`
+		SourceID  string         `json:"source_id"`
+		Index     int            `json:"index"`
+		Ref       SourceRef      `json:"ref"`
+		Content   []byte         `json:"content"`
+		MediaType string         `json:"media_type"`
+		Units     []SourceUnit   `json:"units"`
+		Metadata  map[string]any `json:"metadata,omitempty"`
+	}{
+		ID:        chunk.ID,
+		SourceID:  chunk.SourceID,
+		Index:     chunk.Index,
+		Ref:       chunk.Ref,
+		Content:   chunk.Content,
+		MediaType: chunk.MediaType,
+		Units:     chunk.Units,
+		Metadata:  chunk.Metadata,
+	}
+	encoded, err := json.Marshal(payload)
+	if err != nil {
+		return "", fmt.Errorf("encode source chunk for digest: %w", err)
+	}
+	sum := sha256.Sum256(encoded)
+	return "sha256:" + hex.EncodeToString(sum[:]), nil
+}

internal/core/source/source.go

@@ -22,3 +22,14 @@ type SourceRef struct {
 	StartUnitID int    `json:"start_unit_id"`
 	EndUnitID   int    `json:"end_unit_id"`
 }
+
+type Chunk struct {
+	ID        string         `json:"id"`
+	SourceID  string         `json:"source_id"`
+	Index     int            `json:"index"`
+	Ref       SourceRef      `json:"ref"`
+	Content   []byte         `json:"-"`
+	MediaType string         `json:"media_type"`
+	Units     []SourceUnit   `json:"units"`
+	Metadata  map[string]any `json:"metadata,omitempty"`
+}

internal/core/source/source_test.go

@@ -222,6 +222,42 @@ func TestDigestDocumentIsDeterministicAndIncludesUnitReference(t *testing.T) {
 	}
 }
 
+func TestDigestChunkIsDeterministicAndIncludesReference(t *testing.T) {
+	doc := validDocument()
+	chunk := Chunk{
+		ID:        "chunk-1",
+		SourceID:  doc.ID,
+		Index:     0,
+		Ref:       SourceRef{SourceID: doc.ID, StartUnitID: 1, EndUnitID: 2},
+		Content:   []byte("chunk content"),
+		MediaType: "text/plain",
+		Units:     doc.Units,
+		Metadata:  map[string]any{"second": "value", "first": true},
+	}
+	first, err := DigestChunk(chunk)
+	if err != nil {
+		t.Fatalf("DigestChunk() error = %v, want nil", err)
+	}
+
+	chunk.Metadata = map[string]any{"first": true, "second": "value"}
+	second, err := DigestChunk(chunk)
+	if err != nil {
+		t.Fatalf("DigestChunk(reordered metadata) error = %v, want nil", err)
+	}
+	if first != second {
+		t.Fatalf("digests = %q and %q, want deterministic map ordering", first, second)
+	}
+
+	chunk.Ref.EndUnitID = 1
+	changed, err := DigestChunk(chunk)
+	if err != nil {
+		t.Fatalf("DigestChunk(changed ref) error = %v, want nil", err)
+	}
+	if first == changed {
+		t.Fatalf("digest = %q after reference change, want different digest", changed)
+	}
+}
+
 func TestValidateRefValid(t *testing.T) {
 	doc := validDocument()
 	ref := SourceRef{