Generate and materialize canonical chunk plans

2026-07-17 23:57:59 +00:00
parent 3bfac14397
commit 7844c0a93f
23 changed files with 396 additions and 540 deletions
--- a/internal/framework/contracts/contracts.go
+++ b/internal/framework/contracts/contracts.go
@@ -146,15 +146,15 @@ type ChunkRequest struct {
 	Metadata    map[string]any         `json:"metadata,omitempty"`
 }

-type ChunkResult struct {
-	Chunks   []source.Chunk `json:"chunks"`
-	Warnings []Warning      `json:"warnings,omitempty"`
+type ChunkPlanResult struct {
+	Plan     source.ChunkPlan `json:"plan"`
+	Warnings []Warning        `json:"warnings,omitempty"`
 }

 type Chunker interface {
 	Key() string
 	ReferenceSlots() []ReferenceSlot
-	Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error)
+	Plan(ctx context.Context, req ChunkRequest) (ChunkPlanResult, error)
 }

 const (
--- a/internal/framework/contracts/contracts_test.go
+++ b/internal/framework/contracts/contracts_test.go
@@ -45,7 +45,7 @@ func TestFakeExtractorReturnsTypedOutput(t *testing.T) {
 	}
 }

-func TestFakeChunkerReturnsSourceChunks(t *testing.T) {
+func TestFakeChunkerReturnsSourcePlan(t *testing.T) {
 	doc := &source.SourceDocument{
 		ID:     "source-1",
 		Kind:   "document",
@@ -57,36 +57,19 @@ func TestFakeChunkerReturnsSourceChunks(t *testing.T) {
 	}
 	chunker := fakeChunker{key: "generic-chunker"}

-	result, err := chunker.Chunk(context.Background(), ChunkRequest{Source: doc})
+	result, err := chunker.Plan(context.Background(), ChunkRequest{Source: doc})
 	if err != nil {
-		t.Fatalf("Chunk() error = %v, want nil", err)
+		t.Fatalf("Plan() error = %v, want nil", err)
 	}

 	if chunker.Key() != "generic-chunker" {
 		t.Fatalf("Key() = %q, want generic-chunker", chunker.Key())
 	}
-	if len(result.Chunks) != 1 {
-		t.Fatalf("len(Chunks) = %d, want 1", len(result.Chunks))
+	if len(result.Plan.Ranges) != 1 {
+		t.Fatalf("len(Ranges) = %d, want 1", len(result.Plan.Ranges))
 	}
-
-	chunk := result.Chunks[0]
-	if chunk.ID != "source-1:chunk:0" {
-		t.Fatalf("source.Chunk.ID = %q, want source-1:chunk:0", chunk.ID)
-	}
-	if chunk.SourceID != doc.ID {
-		t.Fatalf("source.Chunk.SourceID = %q, want %q", chunk.SourceID, doc.ID)
-	}
-	if chunk.Index != 0 {
-		t.Fatalf("source.Chunk.Index = %d, want 0", chunk.Index)
-	}
-	if chunk.Ref.StartUnitID != 1 || chunk.Ref.EndUnitID != 1 {
-		t.Fatalf("source.Chunk.Ref = %#v, want source-1:1-1", chunk.Ref)
-	}
-	if chunk.MediaType != "application/json" || string(chunk.Content) != `{"units":[{"id":1,"kind":"section","text":"Source text."}]}` {
-		t.Fatalf("source.Chunk payload = %q %s, want JSON units", chunk.MediaType, chunk.Content)
-	}
-	if len(chunk.Units) != 1 {
-		t.Fatalf("len(source.Chunk.Units) = %d, want 1", len(chunk.Units))
+	if result.Plan.SourceDigest != doc.Digest || result.Plan.Ranges[0].StartUnitID != 1 || result.Plan.Ranges[0].EndUnitID != 1 {
+		t.Fatalf("Plan = %#v, want source digest and unit range", result.Plan)
 	}
 }

@@ -102,8 +85,8 @@ func TestFakeChunkerReceivesPerRunContext(t *testing.T) {
 	}
 	chunker := &recordingChunker{key: "llm-chunker"}

-	if _, err := chunker.Chunk(context.Background(), ChunkRequest{Source: doc, SessionID: "session", LLMProfile: "profile"}); err != nil {
-		t.Fatalf("Chunk() error = %v, want nil", err)
+	if _, err := chunker.Plan(context.Background(), ChunkRequest{Source: doc, SessionID: "session", LLMProfile: "profile"}); err != nil {
+		t.Fatalf("Plan() error = %v, want nil", err)
 	}
 	if chunker.request.SessionID != "session" || chunker.request.LLMProfile != "profile" {
 		t.Fatalf("ChunkRequest = %#v, want per-run session and profile", chunker.request)
@@ -446,22 +429,14 @@ func (chunker fakeChunker) ReferenceSlots() []ReferenceSlot {
 	return nil
 }

-func (chunker fakeChunker) Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error) {
-	return ChunkResult{
-		Chunks: []source.Chunk{
-			{
-				ID:       req.Source.ID + ":chunk:0",
-				SourceID: req.Source.ID,
-				Index:    0,
-				Ref: source.SourceRef{
-					SourceID:    req.Source.ID,
-					StartUnitID: req.Source.Units[0].ID,
-					EndUnitID:   req.Source.Units[len(req.Source.Units)-1].ID,
-				},
-				Content:   []byte(`{"units":[{"id":1,"kind":"section","text":"Source text."}]}`),
-				MediaType: "application/json",
-				Units:     append([]source.SourceUnit(nil), req.Source.Units...),
-			},
+func (chunker fakeChunker) Plan(ctx context.Context, req ChunkRequest) (ChunkPlanResult, error) {
+	return ChunkPlanResult{
+		Plan: source.ChunkPlan{
+			SourceDigest: req.Source.Digest,
+			Ranges: []source.ChunkRange{{
+				StartUnitID: req.Source.Units[0].ID,
+				EndUnitID:   req.Source.Units[len(req.Source.Units)-1].ID,
+			}},
 		},
 	}, nil
 }
@@ -479,9 +454,9 @@ func (chunker *recordingChunker) ReferenceSlots() []ReferenceSlot {
 	return nil
 }

-func (chunker *recordingChunker) Chunk(ctx context.Context, req ChunkRequest) (ChunkResult, error) {
+func (chunker *recordingChunker) Plan(ctx context.Context, req ChunkRequest) (ChunkPlanResult, error) {
 	chunker.request = req
-	return fakeChunker{key: chunker.key}.Chunk(ctx, req)
+	return fakeChunker{key: chunker.key}.Plan(ctx, req)
 }

 type fakeExtractor struct {
--- a/internal/framework/pipeline/chunk_annotations_test.go
+++ b/internal/framework/pipeline/chunk_annotations_test.go
@@ -10,27 +10,29 @@ import (

 func TestChunkCanonicalizationAndClonePreserveAnnotationScopes(t *testing.T) {
 	doc := validSourceDocument()
-	chunk := source.Chunk{
-		ID: "chunk-1", SourceID: doc.ID, Index: 0,
-		Ref: doc.Units[0].Ref, Content: []byte(`{"units":[1]}`), MediaType: "application/json", Units: doc.Units[:1],
-		Annotations:     source.ChunkAnnotations{"shared": json.RawMessage(` {"range": true} `)},
-		PlanAnnotations: source.ChunkAnnotations{"shared": json.RawMessage(` {"plan": true} `)},
+	plan := source.ChunkPlan{
+		SourceDigest: doc.Digest,
+		Ranges: []source.ChunkRange{{
+			StartUnitID: doc.Units[0].ID, EndUnitID: doc.Units[0].ID,
+			Annotations: source.ChunkAnnotations{"shared": json.RawMessage(` {"range": true} `)},
+		}},
+		Annotations: source.ChunkAnnotations{"shared": json.RawMessage(` {"plan": true} `)},
 	}
-	canonical, err := validateAndCanonicalizeChunkResult(doc, []source.Chunk{chunk})
+	canonical, chunks, err := validateAndMaterializeChunkPlan(doc, plan)
 	if err != nil {
-		t.Fatalf("validateAndCanonicalizeChunkResult() error = %v", err)
+		t.Fatalf("validateAndMaterializeChunkPlan() error = %v", err)
 	}
-	if got := string(canonical[0].Annotations["shared"]); got != `{"range":true}` {
+	if got := string(canonical.Ranges[0].Annotations["shared"]); got != `{"range":true}` {
 		t.Fatalf("range annotation = %q", got)
 	}
-	if got := string(canonical[0].PlanAnnotations["shared"]); got != `{"plan":true}` {
+	if got := string(canonical.Annotations["shared"]); got != `{"plan":true}` {
 		t.Fatalf("plan annotation = %q", got)
 	}

-	cloned := cloneSourceChunk(canonical[0])
+	cloned := cloneSourceChunk(chunks[0])
 	cloned.Annotations["shared"][0] = '['
 	cloned.PlanAnnotations["shared"][0] = '['
-	if string(canonical[0].Annotations["shared"]) != `{"range":true}` || string(canonical[0].PlanAnnotations["shared"]) != `{"plan":true}` {
+	if string(chunks[0].Annotations["shared"]) != `{"range":true}` || string(chunks[0].PlanAnnotations["shared"]) != `{"plan":true}` {
 		t.Fatal("cloneSourceChunk() shares annotation bytes")
 	}
 }
--- a/internal/framework/pipeline/chunk_validation.go
+++ b/internal/framework/pipeline/chunk_validation.go
@@ -2,110 +2,23 @@ package pipeline

 import (
 	"fmt"
-	"strings"

 	"gitea.maximumdirect.net/eric/notarius/internal/core/source"
 )

-func validateAndCanonicalizeChunkResult(doc *source.SourceDocument, chunks []source.Chunk) ([]source.Chunk, error) {
-	if len(chunks) == 0 {
-		return nil, fmt.Errorf("chunks must not be empty")
+func validateAndMaterializeChunkPlan(doc *source.SourceDocument, plan source.ChunkPlan) (source.ChunkPlan, []source.Chunk, error) {
+	canonical, err := source.CanonicalizeChunkPlan(plan)
+	if err != nil {
+		return source.ChunkPlan{}, nil, fmt.Errorf("canonicalize chunk plan: %w", err)
 	}
-
-	sourceUnitIndexes := make(map[int]int, len(doc.Units))
-	sourceUnits := make(map[int]source.SourceUnit, len(doc.Units))
-	for index, unit := range doc.Units {
-		sourceUnitIndexes[unit.ID] = index
-		sourceUnits[unit.ID] = unit
+	if err := source.ValidateChunkPlan(doc, canonical); err != nil {
+		return source.ChunkPlan{}, nil, err
 	}
-
-	canonicalChunks := make([]source.Chunk, 0, len(chunks))
-	seenChunkIDs := make(map[string]struct{}, len(chunks))
-	for chunkIndex, chunk := range chunks {
-		if strings.TrimSpace(chunk.ID) == "" {
-			return nil, fmt.Errorf("chunk[%d].id must not be empty", chunkIndex)
-		}
-		if _, ok := seenChunkIDs[chunk.ID]; ok {
-			return nil, fmt.Errorf("chunk id %q is duplicated", chunk.ID)
-		}
-		seenChunkIDs[chunk.ID] = struct{}{}
-
-		if chunk.SourceID != doc.ID {
-			return nil, fmt.Errorf("chunk %q source_id %q does not match source document id %q", chunk.ID, chunk.SourceID, doc.ID)
-		}
-		if chunk.Index != chunkIndex {
-			return nil, fmt.Errorf("chunk %q index %d does not match returned order %d", chunk.ID, chunk.Index, chunkIndex)
-		}
-		if len(chunk.Units) == 0 {
-			return nil, fmt.Errorf("chunk %q units must not be empty", chunk.ID)
-		}
-		if len(chunk.Content) == 0 {
-			return nil, fmt.Errorf("chunk %q content must not be empty", chunk.ID)
-		}
-		if strings.TrimSpace(chunk.MediaType) == "" {
-			return nil, fmt.Errorf("chunk %q media_type must not be empty", chunk.ID)
-		}
-		if err := source.ValidateRef(doc, chunk.Ref); err != nil {
-			return nil, fmt.Errorf("chunk %q ref: %w", chunk.ID, err)
-		}
-		annotations, err := source.CanonicalizeChunkAnnotations(chunk.Annotations)
-		if err != nil {
-			return nil, fmt.Errorf("chunk %q annotations: %w", chunk.ID, err)
-		}
-		planAnnotations, err := source.CanonicalizeChunkAnnotations(chunk.PlanAnnotations)
-		if err != nil {
-			return nil, fmt.Errorf("chunk %q plan_annotations: %w", chunk.ID, err)
-		}
-
-		seenUnitIDs := make(map[int]struct{}, len(chunk.Units))
-		previousSourceIndex := -1
-		canonicalUnits := make([]source.SourceUnit, 0, len(chunk.Units))
-		for unitIndex, unit := range chunk.Units {
-			if unit.ID <= 0 {
-				return nil, fmt.Errorf("chunk %q unit[%d].id must be positive", chunk.ID, unitIndex)
-			}
-			if _, ok := seenUnitIDs[unit.ID]; ok {
-				return nil, fmt.Errorf("chunk %q repeats source unit %d", chunk.ID, unit.ID)
-			}
-			seenUnitIDs[unit.ID] = struct{}{}
-
-			sourceIndex, ok := sourceUnitIndexes[unit.ID]
-			if !ok {
-				return nil, fmt.Errorf("chunk %q source unit %d was not found in source document %q", chunk.ID, unit.ID, doc.ID)
-			}
-			if previousSourceIndex >= 0 && sourceIndex != previousSourceIndex+1 {
-				return nil, fmt.Errorf("chunk %q source units must form a contiguous range in source document order", chunk.ID)
-			}
-			if unit.Ref != sourceUnits[unit.ID].Ref {
-				return nil, fmt.Errorf("chunk %q source unit %d ref does not match source document", chunk.ID, unit.ID)
-			}
-			previousSourceIndex = sourceIndex
-			canonicalUnits = append(canonicalUnits, cloneSourceUnit(sourceUnits[unit.ID]))
-		}
-		expectedRef := source.SourceRef{
-			SourceID:    doc.ID,
-			StartUnitID: canonicalUnits[0].Ref.StartUnitID,
-			EndUnitID:   canonicalUnits[len(canonicalUnits)-1].Ref.EndUnitID,
-		}
-		if chunk.Ref != expectedRef {
-			return nil, fmt.Errorf("chunk %q ref %#v does not match unit span %#v", chunk.ID, chunk.Ref, expectedRef)
-		}
-
-		canonicalChunks = append(canonicalChunks, source.Chunk{
-			ID:              chunk.ID,
-			SourceID:        chunk.SourceID,
-			Index:           chunk.Index,
-			Ref:             expectedRef,
-			Content:         append([]byte(nil), chunk.Content...),
-			MediaType:       chunk.MediaType,
-			Units:           canonicalUnits,
-			Metadata:        cloneMetadata(chunk.Metadata),
-			Annotations:     annotations,
-			PlanAnnotations: planAnnotations,
-		})
+	chunks, err := source.MaterializeChunkPlan(doc, canonical)
+	if err != nil {
+		return source.ChunkPlan{}, nil, err
 	}
-
-	return canonicalChunks, nil
+	return canonical, chunks, nil
 }

 func cloneSourceUnit(unit source.SourceUnit) source.SourceUnit {
--- a/internal/framework/pipeline/chunker_registry_test.go
+++ b/internal/framework/pipeline/chunker_registry_test.go
@@ -334,8 +334,8 @@ func (chunker registryChunker) ReferenceSlots() []contracts.ReferenceSlot {
 	return nil
 }

-func (chunker registryChunker) Chunk(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkResult, error) {
-	return contracts.ChunkResult{}, nil
+func (chunker registryChunker) Plan(ctx context.Context, req contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
+	return contracts.ChunkPlanResult{}, nil
 }

 type registryOutputEncoder struct {
--- a/internal/framework/pipeline/debug.go
+++ b/internal/framework/pipeline/debug.go
@@ -111,6 +111,18 @@ type debugSourceChunk struct {
 	PlanAnnotations source.ChunkAnnotations `json:"plan_annotations,omitempty"`
 }

+type debugChunkPlan struct {
+	SourceDigest string            `json:"source_digest,omitempty"`
+	Ranges       []debugChunkRange `json:"ranges,omitempty"`
+	Annotations  map[string]any    `json:"annotations,omitempty"`
+}
+
+type debugChunkRange struct {
+	StartUnitID int            `json:"start_unit_id"`
+	EndUnitID   int            `json:"end_unit_id"`
+	Annotations map[string]any `json:"annotations,omitempty"`
+}
+
 type debugSerializedOutput struct {
 	LaneID        string                   `json:"lane_id"`
 	NormalizerKey string                   `json:"normalizer_key"`
@@ -489,6 +501,37 @@ func debugSourceChunkEnvelopes(chunks []source.Chunk) []debugSourceChunk {
 	return out
 }

+func debugChunkPlanEnvelope(plan source.ChunkPlan) debugChunkPlan {
+	envelope := debugChunkPlan{
+		SourceDigest: plan.SourceDigest,
+		Ranges:       make([]debugChunkRange, len(plan.Ranges)),
+		Annotations:  debugChunkAnnotations(plan.Annotations),
+	}
+	for i, chunkRange := range plan.Ranges {
+		envelope.Ranges[i] = debugChunkRange{
+			StartUnitID: chunkRange.StartUnitID,
+			EndUnitID:   chunkRange.EndUnitID,
+			Annotations: debugChunkAnnotations(chunkRange.Annotations),
+		}
+	}
+	return envelope
+}
+
+func debugChunkAnnotations(annotations source.ChunkAnnotations) map[string]any {
+	if len(annotations) == 0 {
+		return nil
+	}
+	out := make(map[string]any, len(annotations))
+	for namespace, raw := range annotations {
+		if json.Valid(raw) {
+			out[namespace] = append(json.RawMessage(nil), raw...)
+		} else {
+			out[namespace] = string(raw)
+		}
+	}
+	return out
+}
+
 func debugSerializedOutputEnvelope(output contracts.SerializedOutput) debugSerializedOutput {
 	schema := contracts.CloneArtifactSchema(output.Artifact.Schema)
 	digest := contracts.DigestArtifactSchema(schema)
--- a/internal/framework/pipeline/runner.go
+++ b/internal/framework/pipeline/runner.go
@@ -175,6 +175,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
 	chunker := input.Prepared.chunker
 	attachModuleManifestMetadata(&output, "chunker", chunker)
 	var canonicalChunks []source.Chunk
+	var generatedPlan *source.ChunkPlan
 	var chunkWarnings []contracts.Warning
 	chunkCheckpoint, chunkDecision := checkpointLoader.Chunk(chunker.Key(), doc.Digest)
 	recordCheckpointEvent(&output, checkpointLoader, string(StageChunk), "", chunker.Key(), chunkDecision)
@@ -209,7 +210,7 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
 			attemptPath := path.Join("chunk", fmt.Sprintf("attempt-%02d", attempt))
 			attemptCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
 			terminal := newAttemptTerminalRecorder(debugRecorder, attemptPath, "chunk", llmScope, debugTimedEnvelope{Stage: string(StageChunk), ModuleKey: chunker.Key(), Attempt: attempt, StartedAt: attemptStarted})
-			chunkResult, err := chunker.Chunk(attemptCtx, contracts.ChunkRequest{
+			chunkResult, err := chunker.Plan(attemptCtx, contracts.ChunkRequest{
 				Source:      doc,
 				SourceInput: sourceInput.Clone(),
 				SessionID:   sessionID,
@@ -221,27 +222,25 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
 				attemptErr := fmt.Errorf("chunk source with chunker %q: %w", chunker.Key(), err)
 				return false, nil, terminal.record(nil, attemptErr)
 			}
-			if len(chunkResult.Chunks) == 0 {
-				attemptErr := fmt.Errorf("chunker %q returned no chunks", chunker.Key())
-				return false, nil, terminal.record(nil, attemptErr)
-			}
-			chunks, err := validateAndCanonicalizeChunkResult(doc, chunkResult.Chunks)
+			plan, chunks, err := validateAndMaterializeChunkPlan(doc, chunkResult.Plan)
 			if err != nil {
-				attemptErr := fmt.Errorf("validate chunks from chunker %q: %w", chunker.Key(), err)
-				payload := map[string]any{"warnings": debugWarningEnvelopes(chunkResult.Warnings)}
+				attemptErr := fmt.Errorf("validate chunk plan from chunker %q: %w", chunker.Key(), err)
+				payload := map[string]any{"plan": debugChunkPlanEnvelope(chunkResult.Plan), "warnings": debugWarningEnvelopes(chunkResult.Warnings)}
 				return false, nil, terminal.record(payload, attemptErr)
 			}
 			validationWarnings, rejection, err := r.validateChunks(attemptCtx, doc, chunker.Key(), chunks, sourceInput, sessionID, input.pipeline.ChunkReferences.ReferenceSet, input.Metadata, input.Prepared.chunkValidators, attempt, input.Debug)
 			attemptWarnings := append(cloneWarnings(chunkResult.Warnings), validationWarnings...)
 			payload := map[string]any{
-				"chunks":    debugSourceChunkEnvelopes(chunks),
-				"warnings":  debugWarningEnvelopes(attemptWarnings),
-				"rejection": debugRejectedOutputPtr(rejection),
+				"plan":                debugChunkPlanEnvelope(plan),
+				"materialized_chunks": debugSourceChunkEnvelopes(chunks),
+				"warnings":            debugWarningEnvelopes(attemptWarnings),
+				"rejection":           debugRejectedOutputPtr(rejection),
 			}
 			if err != nil || rejection != nil {
 				return false, rejection, terminal.record(payload, err)
 			}
 			canonicalChunks = chunks
+			generatedPlan = &plan
 			chunkWarnings = attemptWarnings
 			if err := terminal.record(payload, nil); err != nil {
 				return false, nil, err
@@ -265,10 +264,13 @@ func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err
 		}
 	}
 	chunkDebugPayload := map[string]any{
-		"reused":   chunkDecision.Reused,
-		"accepted": chunksAccepted,
-		"chunks":   debugSourceChunkEnvelopes(canonicalChunks),
-		"warnings": chunkWarnings,
+		"reused":              chunkDecision.Reused,
+		"accepted":            chunksAccepted,
+		"materialized_chunks": debugSourceChunkEnvelopes(canonicalChunks),
+		"warnings":            chunkWarnings,
+	}
+	if generatedPlan != nil {
+		chunkDebugPayload["plan"] = debugChunkPlanEnvelope(*generatedPlan)
 	}
 	if chunkRejection != nil {
 		chunkDebugPayload["rejection"] = debugRejectedOutputEnvelope(*chunkRejection)
--- a/internal/framework/pipeline/runner_concurrency_test.go
+++ b/internal/framework/pipeline/runner_concurrency_test.go
@@ -11,7 +11,6 @@ import (
 	"testing"
 	"time"

-	"gitea.maximumdirect.net/eric/notarius/internal/core/source"
 	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
 	frameworkllm "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
 )
@@ -51,12 +50,13 @@ func preparedConcurrentPipeline(t *testing.T, chunkCount int) *PreparedPipeline
 	if err != nil {
 		t.Fatalf("Prepare() error = %v", err)
 	}
-	doc := typedTestDocument()
-	chunks := make([]source.Chunk, chunkCount)
-	for i := range chunks {
-		chunks[i] = source.Chunk{ID: fmt.Sprintf("chunk-%d", i+1), SourceID: doc.ID, Index: i, Ref: doc.Units[0].Ref, Content: []byte(fmt.Sprintf(`{"chunk":%d}`, i+1)), MediaType: "application/json", Units: []source.SourceUnit{doc.Units[0]}}
+	doc := typedTestDocumentWithUnits(chunkCount)
+	if adapter, ok := prepared.input.(*typedTestInput); ok {
+		adapter.doc = doc
+	} else {
+		t.Fatalf("prepared input = %T, want *typedTestInput", prepared.input)
 	}
-	prepared.chunker = &typedTestChunker{key: "typed/chunk", chunks: chunks}
+	prepared.chunker = &typedTestChunker{key: "typed/chunk", plan: typedTestPlan(doc)}
 	return prepared
 }

--- a/internal/framework/pipeline/runner_extract_handoff_test.go
+++ b/internal/framework/pipeline/runner_extract_handoff_test.go
@@ -55,8 +55,8 @@ func cloneCheckpointArtifacts(values []CheckpointArtifact) []CheckpointArtifact
 func TestRunnerPassesIndependentAnnotationScopesToExtractors(t *testing.T) {
 	prepared := preparedAttemptDebugPipeline(t)
 	chunker := prepared.chunker.(*typedTestChunker)
-	chunker.chunks[0].Annotations = source.ChunkAnnotations{"same": json.RawMessage(`{"range":1}`)}
-	chunker.chunks[0].PlanAnnotations = source.ChunkAnnotations{"same": json.RawMessage(`{"plan":2}`)}
+	chunker.plan.Ranges[0].Annotations = source.ChunkAnnotations{"same": json.RawMessage(`{"range":1}`)}
+	chunker.plan.Annotations = source.ChunkAnnotations{"same": json.RawMessage(`{"plan":2}`)}

 	var gotRange, gotPlan string
 	installExtractOperation(prepared, 0, func(_ context.Context, request contracts.TypedExtractionRequest) (erasedTypedResult, error) {
@@ -72,7 +72,7 @@ func TestRunnerPassesIndependentAnnotationScopesToExtractors(t *testing.T) {
 	if gotRange != `{"range":1}` || gotPlan != `{"plan":2}` {
 		t.Fatalf("extract annotations = %q / %q", gotRange, gotPlan)
 	}
-	if string(chunker.chunks[0].Annotations["same"]) != `{"range":1}` || string(chunker.chunks[0].PlanAnnotations["same"]) != `{"plan":2}` {
+	if string(chunker.plan.Ranges[0].Annotations["same"]) != `{"range":1}` || string(chunker.plan.Annotations["same"]) != `{"plan":2}` {
 		t.Fatal("extract request shares annotation bytes with chunker output")
 	}
 }
--- a/internal/framework/pipeline/runner_terminal_debug_test.go
+++ b/internal/framework/pipeline/runner_terminal_debug_test.go
@@ -15,7 +15,7 @@ import (

 type terminalChunker struct {
 	key      string
-	chunks   []source.Chunk
+	plan     source.ChunkPlan
 	warnings []contracts.Warning
 	err      error
 	calls    *int
@@ -25,11 +25,11 @@ func (c terminalChunker) Key() string { return c.key }

 func (terminalChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }

-func (c terminalChunker) Chunk(context.Context, contracts.ChunkRequest) (contracts.ChunkResult, error) {
+func (c terminalChunker) Plan(context.Context, contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
 	if c.calls != nil {
 		(*c.calls)++
 	}
-	return contracts.ChunkResult{Chunks: cloneSourceChunks(c.chunks), Warnings: cloneWarnings(c.warnings)}, c.err
+	return contracts.ChunkPlanResult{Plan: source.CloneChunkPlan(c.plan), Warnings: cloneWarnings(c.warnings)}, c.err
 }

 type terminalChunkValidator struct {
@@ -37,6 +37,19 @@ type terminalChunkValidator struct {
 	err    error
 }

+type observingChunkValidator struct {
+	request contracts.ChunkValidationRequest
+}
+
+func (*observingChunkValidator) Name() string { return "observing/chunk-validator" }
+func (*observingChunkValidator) ExecutionClass() contracts.ExecutionClass {
+	return contracts.ExecutionClassDeterministic
+}
+func (v *observingChunkValidator) Validate(_ context.Context, request contracts.ChunkValidationRequest) (contracts.ValidationResult, error) {
+	v.request = request
+	return contracts.ValidationResult{Approved: true}, nil
+}
+
 func (terminalChunkValidator) Name() string { return "terminal/chunk-validator" }

 func (terminalChunkValidator) ExecutionClass() contracts.ExecutionClass {
@@ -71,14 +84,14 @@ func assertAttemptEnvelopeSequence(t *testing.T, debug *capturedDebugRecorder, p
 	}
 }

-func preparedTerminalDebugPipeline(t *testing.T) (*PreparedPipeline, []source.Chunk) {
+func preparedTerminalDebugPipeline(t *testing.T) (*PreparedPipeline, source.ChunkPlan) {
 	t.Helper()
 	prepared := preparedAttemptDebugPipeline(t)
 	chunker, ok := prepared.chunker.(*typedTestChunker)
 	if !ok {
 		t.Fatalf("prepared chunker = %T, want *typedTestChunker", prepared.chunker)
 	}
-	return prepared, cloneSourceChunks(chunker.chunks)
+	return prepared, source.CloneChunkPlan(chunker.plan)
 }

 func TestRunnerRecordsChunkTerminalOutcomes(t *testing.T) {
@@ -96,8 +109,8 @@ func TestRunnerRecordsChunkTerminalOutcomes(t *testing.T) {
 	}
 	for _, tc := range tests {
 		t.Run(tc.name, func(t *testing.T) {
-			prepared, chunks := preparedTerminalDebugPipeline(t)
-			prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, chunks: chunks, warnings: []contracts.Warning{{Scope: "chunk", ReasonCode: "observed", Message: "chunk warning"}}, err: tc.moduleError}
+			prepared, plan := preparedTerminalDebugPipeline(t)
+			prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, warnings: []contracts.Warning{{Scope: "chunk", ReasonCode: "observed", Message: "chunk warning"}}, err: tc.moduleError}
 			prepared.chunkValidators.validators = []preparedValidator{{resolved: ResolvedValidator{Binding: Binding("terminal/chunk-validator"), Target: ValidatorTargetChunk}, chunk: tc.validator}}
 			debug := newCapturedDebugRecorder()

@@ -116,10 +129,63 @@ func TestRunnerRecordsChunkTerminalOutcomes(t *testing.T) {
 					t.Fatalf("chunk rejection = envelope %#v, outputs %#v", envelope, output.Rejected)
 				}
 			}
+			if tc.name == "accepted" {
+				encoded := string(debug.json["chunk/attempt-01.json"])
+				if !strings.Contains(encoded, `"plan":`) || !strings.Contains(encoded, `"materialized_chunks":`) || strings.Contains(encoded, `"chunks":`) {
+					t.Fatalf("chunk attempt debug = %s, want separate plan and materialized_chunks", encoded)
+				}
+			}
 		})
 	}
 }

+func TestRunnerMaterializesAnnotatedPlanBeforeChunkValidation(t *testing.T) {
+	prepared, plan := preparedTerminalDebugPipeline(t)
+	plan.Annotations = source.ChunkAnnotations{"same": []byte(`{"plan":1}`)}
+	plan.Ranges[0].Annotations = source.ChunkAnnotations{"same": []byte(`{"range":2}`)}
+	prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan}
+	validator := &observingChunkValidator{}
+	prepared.chunkValidators.validators = []preparedValidator{{
+		resolved: ResolvedValidator{Binding: Binding(validator.Name()), Target: ValidatorTargetChunk},
+		chunk:    validator,
+	}}
+
+	if _, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input")}); err != nil {
+		t.Fatalf("Run() error = %v, want nil", err)
+	}
+	if len(validator.request.Chunks) != 1 {
+		t.Fatalf("validator chunks = %#v, want one", validator.request.Chunks)
+	}
+	chunk := validator.request.Chunks[0]
+	if chunk.ID != "chunk-000001" || chunk.Index != 0 || chunk.MediaType != "application/json" {
+		t.Fatalf("materialized chunk identity = %#v", chunk)
+	}
+	if string(chunk.Annotations["same"]) != `{"range":2}` || string(chunk.PlanAnnotations["same"]) != `{"plan":1}` {
+		t.Fatalf("materialized annotations = %s / %s", chunk.Annotations["same"], chunk.PlanAnnotations["same"])
+	}
+	if chunk.Metadata["start_unit_id"] != 1 || chunk.Metadata["end_unit_id"] != 1 || chunk.Metadata["unit_count"] != 1 {
+		t.Fatalf("materialized metadata = %#v", chunk.Metadata)
+	}
+}
+
+func TestRunnerRetriesMalformedPlanWithDebugEnabled(t *testing.T) {
+	prepared, plan := preparedTerminalDebugPipeline(t)
+	plan.Ranges[0].Annotations = source.ChunkAnnotations{"broken": []byte(`{"value":`)}
+	prepared.resolved.Chunk.Retries = 1
+	calls := 0
+	prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}
+	debug := newCapturedDebugRecorder()
+
+	_, err := New().Run(context.Background(), RunInput{Prepared: prepared, RawInput: []byte("input"), Debug: debug})
+	if err == nil || !strings.Contains(err.Error(), "valid JSON") {
+		t.Fatalf("Run() error = %v, want malformed annotation error", err)
+	}
+	if calls != 2 {
+		t.Fatalf("plan calls = %d, want two attempts", calls)
+	}
+	assertAttemptEnvelopeSequence(t, debug, "chunk", 1, 2)
+}
+
 func TestRunnerRecordsExtractTerminalOutcomes(t *testing.T) {
 	tests := []struct {
 		name          string
@@ -187,9 +253,9 @@ func TestRunnerRecordsExtractTerminalOutcomes(t *testing.T) {

 func TestRunnerJoinsPrimaryAndAttemptWriteErrors(t *testing.T) {
 	t.Run("chunk", func(t *testing.T) {
-		prepared, chunks := preparedTerminalDebugPipeline(t)
+		prepared, plan := preparedTerminalDebugPipeline(t)
 		primary := errors.New("chunk operation failed")
-		prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, chunks: chunks, err: primary}
+		prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, err: primary}
 		debug := newCapturedDebugRecorder()
 		debug.failPath = "chunk/attempt-01.json"

@@ -216,10 +282,10 @@ func TestRunnerJoinsPrimaryAndAttemptWriteErrors(t *testing.T) {
 }

 func TestRunnerDoesNotRetryAfterTerminalAttemptWriteFailure(t *testing.T) {
-	prepared, chunks := preparedTerminalDebugPipeline(t)
+	prepared, plan := preparedTerminalDebugPipeline(t)
 	prepared.resolved.Chunk.Retries = 1
 	calls := 0
-	prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, chunks: chunks, calls: &calls}
+	prepared.chunker = terminalChunker{key: prepared.resolved.Chunk.Module, plan: plan, calls: &calls}
 	prepared.chunkValidators.validators = []preparedValidator{{
 		resolved: ResolvedValidator{Binding: Binding("terminal/chunk-validator"), Target: ValidatorTargetChunk},
 		chunk:    terminalChunkValidator{result: contracts.ValidationResult{Approved: true}},
--- a/internal/framework/pipeline/typed_resolution_test.go
+++ b/internal/framework/pipeline/typed_resolution_test.go
@@ -67,14 +67,14 @@ func (v *typedTestInput) Parse(context.Context, contracts.ParseRequest) (*source
 }

 type typedTestChunker struct {
-	key    string
-	chunks []source.Chunk
+	key  string
+	plan source.ChunkPlan
 }

 func (v *typedTestChunker) Key() string                               { return v.key }
 func (v *typedTestChunker) ReferenceSlots() []contracts.ReferenceSlot { return nil }
-func (v *typedTestChunker) Chunk(context.Context, contracts.ChunkRequest) (contracts.ChunkResult, error) {
-	return contracts.ChunkResult{Chunks: v.chunks}, nil
+func (v *typedTestChunker) Plan(context.Context, contracts.ChunkRequest) (contracts.ChunkPlanResult, error) {
+	return contracts.ChunkPlanResult{Plan: source.CloneChunkPlan(v.plan)}, nil
 }

 type typedTestOutput struct{ key string }
@@ -84,12 +84,27 @@ func (v *typedTestOutput) Encode(context.Context, contracts.OutputRequest) (cont
 	return contracts.OutputResult{}, nil
 }
 func typedTestDocument() *source.SourceDocument {
-	ref := source.SourceRef{SourceID: "source", StartUnitID: 1, EndUnitID: 1}
-	doc := &source.SourceDocument{ID: "source", Kind: "document", Format: "text/plain", Units: []source.SourceUnit{{ID: 1, Kind: "line", Text: "text", Ref: ref}}}
+	return typedTestDocumentWithUnits(1)
+}
+
+func typedTestDocumentWithUnits(count int) *source.SourceDocument {
+	doc := &source.SourceDocument{ID: "source", Kind: "document", Format: "text/plain"}
+	for id := 1; id <= count; id++ {
+		ref := source.SourceRef{SourceID: doc.ID, StartUnitID: id, EndUnitID: id}
+		doc.Units = append(doc.Units, source.SourceUnit{ID: id, Kind: "line", Text: fmt.Sprintf("text-%d", id), Ref: ref})
+	}
 	doc.Digest, _ = source.DigestDocument(doc)
 	return doc
 }

+func typedTestPlan(doc *source.SourceDocument) source.ChunkPlan {
+	plan := source.ChunkPlan{SourceDigest: doc.Digest, Ranges: make([]source.ChunkRange, len(doc.Units))}
+	for i, unit := range doc.Units {
+		plan.Ranges[i] = source.ChunkRange{StartUnitID: unit.ID, EndUnitID: unit.ID}
+	}
+	return plan
+}
+
 func (v typedTestSerializedValidator) Name() string { return v.key }
 func (typedTestSerializedValidator) ExecutionClass() contracts.ExecutionClass {
 	return contracts.ExecutionClassDeterministic
@@ -466,7 +481,7 @@ func mustRegisterTypedTestBase(t *testing.T, catalog ModuleCatalog) {
 	}
 	if err := catalog.Chunkers.RegisterWithSpec(ModuleSpec{Key: "typed/chunk", Stage: StageChunk}, func() (contracts.Chunker, error) {
 		doc := typedTestDocument()
-		return &typedTestChunker{key: "typed/chunk", chunks: []source.Chunk{{ID: "chunk-1", SourceID: doc.ID, Index: 0, Ref: doc.Units[0].Ref, Content: []byte(`{"chunk":1}`), MediaType: "application/json", Units: []source.SourceUnit{doc.Units[0]}}}}, nil
+		return &typedTestChunker{key: "typed/chunk", plan: typedTestPlan(doc)}, nil
 	}); err != nil {
 		t.Fatalf("register chunker: %v", err)
 	}