Add canonical chunk plan source model

internal/core/source/digest.go

@@ -37,24 +37,36 @@ func DigestDocument(doc *SourceDocument) (string, error) {
 // DigestChunk returns a deterministic digest of a chunk, including its source
 // provenance, content, units, and metadata.
 func DigestChunk(chunk Chunk) (string, error) {
+	annotations, err := CanonicalizeChunkAnnotations(chunk.Annotations)
+	if err != nil {
+		return "", fmt.Errorf("canonicalize source chunk annotations: %w", err)
+	}
+	planAnnotations, err := CanonicalizeChunkAnnotations(chunk.PlanAnnotations)
+	if err != nil {
+		return "", fmt.Errorf("canonicalize source chunk plan annotations: %w", err)
+	}
 	payload := struct {
-		ID        string         `json:"id"`
-		SourceID  string         `json:"source_id"`
-		Index     int            `json:"index"`
-		Ref       SourceRef      `json:"ref"`
-		Content   []byte         `json:"content"`
-		MediaType string         `json:"media_type"`
-		Units     []SourceUnit   `json:"units"`
-		Metadata  map[string]any `json:"metadata,omitempty"`
+		ID              string           `json:"id"`
+		SourceID        string           `json:"source_id"`
+		Index           int              `json:"index"`
+		Ref             SourceRef        `json:"ref"`
+		Content         []byte           `json:"content"`
+		MediaType       string           `json:"media_type"`
+		Units           []SourceUnit     `json:"units"`
+		Metadata        map[string]any   `json:"metadata,omitempty"`
+		Annotations     ChunkAnnotations `json:"annotations,omitempty"`
+		PlanAnnotations ChunkAnnotations `json:"plan_annotations,omitempty"`
 	}{
-		ID:        chunk.ID,
-		SourceID:  chunk.SourceID,
-		Index:     chunk.Index,
-		Ref:       chunk.Ref,
-		Content:   chunk.Content,
-		MediaType: chunk.MediaType,
-		Units:     chunk.Units,
-		Metadata:  chunk.Metadata,
+		ID:              chunk.ID,
+		SourceID:        chunk.SourceID,
+		Index:           chunk.Index,
+		Ref:             chunk.Ref,
+		Content:         chunk.Content,
+		MediaType:       chunk.MediaType,
+		Units:           chunk.Units,
+		Metadata:        chunk.Metadata,
+		Annotations:     annotations,
+		PlanAnnotations: planAnnotations,
 	}
 	encoded, err := json.Marshal(payload)
 	if err != nil {
@@ -63,3 +75,28 @@ func DigestChunk(chunk Chunk) (string, error) {
 	sum := sha256.Sum256(encoded)
 	return "sha256:" + hex.EncodeToString(sum[:]), nil
 }
+
+// DigestChunkPlan returns a deterministic digest of the logical plan. Storage
+// schema, producer provenance, warnings, and timestamps are intentionally not
+// part of the digest.
+func DigestChunkPlan(plan ChunkPlan) (string, error) {
+	canonical, err := CanonicalizeChunkPlan(plan)
+	if err != nil {
+		return "", err
+	}
+	if isBlank(canonical.SourceDigest) {
+		return "", fmt.Errorf("chunk plan source_digest must not be empty")
+	}
+	if hasSurroundingWhitespace(canonical.SourceDigest) {
+		return "", fmt.Errorf("chunk plan source_digest must not contain leading or trailing whitespace")
+	}
+	if len(canonical.Ranges) == 0 {
+		return "", fmt.Errorf("chunk plan ranges must not be empty")
+	}
+	encoded, err := json.Marshal(canonical)
+	if err != nil {
+		return "", fmt.Errorf("encode chunk plan for digest: %w", err)
+	}
+	sum := sha256.Sum256(encoded)
+	return "sha256:" + hex.EncodeToString(sum[:]), nil
+}