Expose chunk plan provenance and safe diagnostics

internal/framework/pipeline/runner_chunk_plan.go

@@ -18,6 +18,9 @@ type chunkPlanExecution struct {
 	warnings  []contracts.Warning
 	rejection *contracts.RejectedOutput
 	lookup    ChunkPlanDecision
+	record    *ChunkPlanRecord
+	action    string
+	summary   artifacts.ChunkPlanSummary
 }
 
 func effectiveChunkCacheMode(mode ChunkCacheMode) ChunkCacheMode {
@@ -33,33 +36,50 @@ func effectiveChunkCacheMode(mode ChunkCacheMode) ChunkCacheMode {
 func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.SourceDocument, sourceInput contracts.LLMInputMaterial, sessionID string) (chunkPlanExecution, error) {
 	mode := effectiveChunkCacheMode(input.ChunkCacheMode)
 	chunker := input.Prepared.chunker
-	result := chunkPlanExecution{lookup: ChunkPlanDecision{Reason: "chunk plan lookup skipped"}}
+	result := chunkPlanExecution{
+		lookup: ChunkPlanDecision{Reason: "chunk plan lookup skipped"},
+		summary: artifacts.ChunkPlanSummary{
+			Mode: string(mode), SourceDigest: doc.Digest, RequestedModule: chunker.Key(), LookupStatus: "skipped",
+			LookupReason: "chunk plan lookup skipped", ValidationStatus: "not_run", PublicationStatus: "not_requested",
+		},
+	}
 
 	if mode == ChunkCacheAuto {
 		record, decision, err := input.ChunkPlans.Load(doc.Digest)
 		result.lookup = decision
+		result.summary.LookupStatus = string(decision.Status)
+		result.summary.LookupReason = decision.Reason
 		if err != nil {
+			result.summary.LookupStatus = "skipped"
+			result.summary.LookupReason = "chunk plan lookup failed"
 			return result, fmt.Errorf("load chunk plan: %w", err)
 		}
 		switch decision.Status {
 		case ChunkPlanHit:
 			plan, chunks, validationErr := validateAndMaterializeChunkPlan(doc, record.Plan)
 			if validationErr == nil {
+				result.setCandidate(record, "reused")
 				validationWarnings, rejection, err := r.validateChunks(ctx, doc, chunker.Key(), chunks, sourceInput, sessionID, input.pipeline.ChunkReferences.ReferenceSet, input.Metadata, input.Prepared.chunkValidators, 1, input.Debug)
 				result.plan = &plan
 				result.chunks = chunks
 				result.warnings = append(cloneWarnings(record.Warnings), validationWarnings...)
 				result.rejection = rejection
 				result.accepted = rejection == nil && err == nil
+				result.setValidation(validationWarnings, rejection, err)
 				return result, err
 			}
 			result.lookup = ChunkPlanDecision{Status: ChunkPlanInvalid, Reason: "stored chunk plan cannot be materialized against the current source"}
+			result.summary.LookupStatus = "invalid"
+			result.summary.LookupReason = result.lookup.Reason
 		case ChunkPlanMissing, ChunkPlanInvalid:
 			// Generate below.
 		default:
 			return result, fmt.Errorf("load chunk plan returned unsupported status %q", decision.Status)
 		}
 	}
+	if mode == ChunkCacheAuto || mode == ChunkCacheRefresh {
+		result.summary.PublicationStatus = "not_published"
+	}
 
 	var producerWarnings []contracts.Warning
 	accepted, rejection, err := runWithRetry(ctx, input.pipeline.Chunk.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
@@ -81,6 +101,33 @@ func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.S
 			payload := map[string]any{"plan": debugChunkPlanEnvelope(chunkResult.Plan), "warnings": debugWarningEnvelopes(chunkResult.Warnings)}
 			return false, nil, terminal.record(payload, attemptErr)
 		}
+		planDigest, digestErr := source.DigestChunkPlan(plan)
+		if digestErr != nil {
+			return false, nil, terminal.record(nil, fmt.Errorf("digest generated chunk plan: %w", digestErr))
+		}
+		producerMetadata, _ := moduleManifestMetadata(chunker)
+		profile := ""
+		if provider, ok := chunker.(contracts.ChunkExecutionClassProvider); ok && provider.ExecutionClass() == contracts.ExecutionClassLLMBacked {
+			profile = input.pipeline.Chunk.LLMProfile
+		}
+		candidate := ChunkPlanRecord{
+			SchemaVersion: ChunkPlanSchemaVersion, SourceDigest: doc.Digest, PlanDigest: planDigest,
+			Plan: source.CloneChunkPlan(plan),
+			Producer: ChunkPlanProducer{
+				InputModule: input.Prepared.input.Key(), ChunkModule: chunker.Key(), LLMProfile: profile,
+				References: append([]artifacts.ReferenceProvenance(nil), referenceTargetProvenance(input.pipeline.ChunkReferences)...),
+				Metadata:   cloneMetadata(producerMetadata),
+			},
+			Warnings: cloneWarnings(chunkResult.Warnings), CreatedAt: time.Now().UTC(),
+		}
+		action := "generated"
+		if mode == ChunkCacheRefresh {
+			action = "refreshed"
+		}
+		if mode == ChunkCacheBypass {
+			action = "bypassed"
+		}
+		result.setCandidate(candidate, action)
 		validationWarnings, rejected, validationErr := r.validateChunks(attemptCtx, doc, chunker.Key(), chunks, sourceInput, sessionID, input.pipeline.ChunkReferences.ReferenceSet, input.Metadata, input.Prepared.chunkValidators, attempt, input.Debug)
 		attemptWarnings := append(cloneWarnings(chunkResult.Warnings), validationWarnings...)
 		payload := map[string]any{
@@ -88,18 +135,23 @@ func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.S
 			"warnings": debugWarningEnvelopes(attemptWarnings), "rejection": debugRejectedOutputPtr(rejected),
 		}
 		if validationErr != nil || rejected != nil {
+			result.setValidation(validationWarnings, rejected, validationErr)
 			return false, rejected, terminal.record(payload, validationErr)
 		}
 		result.chunks = chunks
 		result.plan = &plan
 		result.warnings = attemptWarnings
 		producerWarnings = cloneWarnings(chunkResult.Warnings)
+		result.setValidation(validationWarnings, nil, nil)
 		if debugErr := terminal.record(payload, nil); debugErr != nil {
 			return false, nil, debugErr
 		}
 		return true, nil, nil
 	})
 	if err != nil {
+		if result.summary.ValidationStatus == "not_run" {
+			result.summary.ValidationStatus = "error"
+		}
 		return result, err
 	}
 	result.accepted = accepted
@@ -109,29 +161,69 @@ func (r *Runner) runChunkPlan(ctx context.Context, input RunInput, doc *source.S
 	}
 
 	if mode == ChunkCacheAuto || mode == ChunkCacheRefresh {
-		planDigest, err := source.DigestChunkPlan(*result.plan)
-		if err != nil {
-			return result, fmt.Errorf("digest generated chunk plan: %w", err)
-		}
-		producerMetadata, _ := moduleManifestMetadata(chunker)
-		record := ChunkPlanRecord{
-			SchemaVersion: ChunkPlanSchemaVersion,
-			SourceDigest:  doc.Digest,
-			PlanDigest:    planDigest,
-			Plan:          source.CloneChunkPlan(*result.plan),
-			Producer: ChunkPlanProducer{
-				InputModule: input.Prepared.input.Key(),
-				ChunkModule: chunker.Key(),
-				LLMProfile:  input.pipeline.Chunk.LLMProfile,
-				References:  append([]artifacts.ReferenceProvenance(nil), referenceTargetProvenance(input.pipeline.ChunkReferences)...),
-				Metadata:    cloneMetadata(producerMetadata),
-			},
-			Warnings:  cloneWarnings(producerWarnings),
-			CreatedAt: time.Now().UTC(),
-		}
+		record := cloneChunkPlanRecord(*result.record)
+		record.Warnings = cloneWarnings(producerWarnings)
 		if err := input.ChunkPlans.Save(record); err != nil {
 			return result, fmt.Errorf("save chunk plan: %w", err)
 		}
+		result.summary.PublicationStatus = "published"
 	}
 	return result, nil
 }
+
+func (result *chunkPlanExecution) setCandidate(record ChunkPlanRecord, action string) {
+	cloned := cloneChunkPlanRecord(record)
+	result.record = &cloned
+	result.action = action
+	result.summary.Action = action
+	result.summary.SourceDigest = record.SourceDigest
+	result.summary.CandidateDigest = record.PlanDigest
+}
+
+func (result *chunkPlanExecution) setValidation(warnings []contracts.Warning, rejection *contracts.RejectedOutput, err error) {
+	switch {
+	case err != nil:
+		result.summary.ValidationStatus = "error"
+	case rejection != nil:
+		result.summary.ValidationStatus = "rejected"
+	case len(warnings) > 0:
+		result.summary.ValidationStatus = "approved_with_warnings"
+	default:
+		result.summary.ValidationStatus = "approved"
+	}
+}
+
+func cloneChunkPlanRecord(record ChunkPlanRecord) ChunkPlanRecord {
+	record.Plan = source.CloneChunkPlan(record.Plan)
+	record.Producer.References = append([]artifacts.ReferenceProvenance(nil), record.Producer.References...)
+	record.Producer.Metadata = cloneMetadata(record.Producer.Metadata)
+	record.Warnings = cloneWarnings(record.Warnings)
+	return record
+}
+
+func applyChunkPlanExecution(output *RunOutput, result chunkPlanExecution) {
+	if output == nil {
+		return
+	}
+	summary := result.summary
+	output.ChunkPlan = &summary
+	if output.Manifest.ChunkPlan == nil {
+		return
+	}
+	manifest := output.Manifest.ChunkPlan
+	manifest.Action = result.action
+	if result.record == nil {
+		return
+	}
+	record := result.record
+	manifest.SourceDigest = record.SourceDigest
+	manifest.PlanDigest = record.PlanDigest
+	manifest.PlanSchemaVersion = record.SchemaVersion
+	manifest.ProducerInputModule = record.Producer.InputModule
+	manifest.ProducerModule = record.Producer.ChunkModule
+	manifest.ProducerLLMProfile = record.Producer.LLMProfile
+	manifest.ProducerReferences = append([]artifacts.ReferenceProvenance(nil), record.Producer.References...)
+	manifest.ProducerMetadata = cloneMetadata(record.Producer.Metadata)
+	createdAt := record.CreatedAt
+	manifest.CreatedAt = &createdAt
+}