notarius/internal/framework/pipeline/runner.go

package pipeline

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"mime"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
	"gitea.maximumdirect.net/eric/notarius/internal/core/source"
	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
)

type Registries struct {
	Inputs          *InputAdapterRegistry
	Chunkers        *ChunkerRegistry
	ArtifactCodecs  *ArtifactCodecRegistry
	Extractors      *ExtractorRegistry
	Mergers         *MergerRegistry
	Normalizers     *NormalizerRegistry
	Validators      *ValidatorRegistry
	ValidatorChains *ValidatorChainRegistry
	Outputs         *OutputEncoderRegistry
}

type Runner struct{}

func New() *Runner {
	return &Runner{}
}

type RunInput struct {
	Prepared    *PreparedPipeline
	SourceID    string
	Path        string
	RawInput    []byte
	SessionID   string
	RunID       string
	StartedAt   time.Time
	LLMProfiles []artifacts.LLMProfileManifest
	Metadata    map[string]any
	Warnings    []contracts.Warning
	Checkpoints CheckpointRecorder
	Checkpoint  CheckpointLoader
	Debug       DebugRecorder
	// ExtractWorkers bounds run-wide extract jobs. Values less than one use a
	// single worker so direct framework callers retain deterministic behavior.
	ExtractWorkers int

	pipeline        ResolvedPipeline
	llmClient       contracts.StructuredLLMClient
	extractDecision *CheckpointDecision
}

type RunOutput struct {
	Manifest         artifacts.RunManifest        `json:"manifest"`
	NormalizeOutputs []contracts.SerializedOutput `json:"normalize_outputs,omitempty"`
	Rejected         []contracts.RejectedOutput   `json:"rejected,omitempty"`
	Warnings         []contracts.Warning          `json:"warnings,omitempty"`
	OutputFiles      []contracts.OutputFile       `json:"-"`
	CheckpointEvents []CheckpointEvent            `json:"checkpoint_events,omitempty"`
}

func (r *Runner) Run(ctx context.Context, input RunInput) (output RunOutput, err error) {
	if r == nil {
		return output, fmt.Errorf("runner must not be nil")
	}
	if err := validateRunInput(input); err != nil {
		return output, err
	}
	input.pipeline = input.Prepared.resolved
	input.llmClient = input.Prepared.dependencies.LLM

	output.Manifest = manifestFromPipeline(input)
	checkpoints := input.Checkpoints
	if checkpoints == nil {
		checkpoints = NoopCheckpointRecorder()
	}
	checkpointLoader := input.Checkpoint
	if checkpointLoader == nil {
		checkpointLoader = NoopCheckpointLoader()
	}
	debugRecorder := input.Debug
	if debugRecorder == nil {
		debugRecorder = NoopDebugRecorder()
	}
	checkpoints = synchronizedCheckpointRecorder(checkpoints)
	checkpointLoader = synchronizedCheckpointLoader(checkpointLoader)
	debugRecorder = synchronizedDebugRecorder(debugRecorder)
	input.Debug = debugRecorder
	input.llmClient = wrapDebugLLMClient(input.llmClient, debugRecorder)
	defer func() {
		output.Manifest.LLMProfiles = mergeLLMProfileManifests(input.LLMProfiles, llmProfileManifests(input.llmClient))
	}()
	output.Warnings = append(output.Warnings, cloneWarnings(input.Warnings)...)
	if err := writeDebugTimed(debugRecorder, "run.json", debugTimedEnvelope{
		Stage:     "run",
		StartedAt: startedTime(input.StartedAt),
		Payload: map[string]any{
			"pipeline_id":     input.pipeline.ID,
			"pipeline_digest": input.pipeline.Digest,
			"run_id":          output.Manifest.RunID,
		},
	}); err != nil {
		return failOutput(output), fmt.Errorf("write debug run artifact: %w", err)
	}

	adapter := input.Prepared.input
	attachModuleManifestMetadata(&output, "input", adapter)
	sourceCheckpoint, sourceDecision := checkpointLoader.Source(adapter.Key())
	recordCheckpointEvent(&output, checkpointLoader, "source", "", adapter.Key(), sourceDecision)
	doc := sourceCheckpoint.Document
	sourceStarted := time.Now().UTC()
	if err := writeDebugTimed(debugRecorder, "source/input.json", debugTimedEnvelope{
		Stage:     "source",
		ModuleKey: adapter.Key(),
		StartedAt: sourceStarted,
		Payload: debugSourceInput{
			SourceID: input.SourceID,
			Path:     input.Path,
			Raw:      debugContentEnvelope(input.RawInput, sourceInputMediaType(input.Path), nil, nil),
			Options:  redactSensitiveMap(input.pipeline.Input.Options),
			Metadata: redactSensitiveMap(input.Metadata),
		},
	}); err != nil {
		return failOutput(output), fmt.Errorf("write source debug artifact: %w", err)
	}
	if !sourceDecision.Reused {
		if err := checkpoints.SourceRunning(adapter.Key()); err != nil {
			return failOutput(output), fmt.Errorf("write source checkpoint: %w", err)
		}
		doc, err = adapter.Parse(ctx, contracts.ParseRequest{
			SourceID:   input.SourceID,
			Path:       input.Path,
			Raw:        input.RawInput,
			LLMProfile: input.pipeline.Input.LLMProfile,
			Metadata:   input.Metadata,
		})
		if err != nil {
			_ = checkpoints.SourceFailed(adapter.Key(), err)
			return failOutput(output), fmt.Errorf("parse input with adapter %q: %w", adapter.Key(), err)
		}
		if err := source.ValidateDocument(doc); err != nil {
			_ = checkpoints.SourceFailed(adapter.Key(), err)
			return failOutput(output), fmt.Errorf("validate source document: %w", err)
		}
		if err := checkpoints.SourceSucceeded(adapter.Key(), doc); err != nil {
			return failOutput(output), fmt.Errorf("write source checkpoint: %w", err)
		}
	}
	if err := writeDebugTimed(debugRecorder, "source/output.json", debugTimedEnvelope{
		Stage:     "source",
		ModuleKey: adapter.Key(),
		StartedAt: sourceStarted,
		Payload: map[string]any{
			"reused":   sourceDecision.Reused,
			"decision": sourceDecision,
			"document": debugSourceDocumentEnvelope(doc),
		},
	}); err != nil {
		return failOutput(output), fmt.Errorf("write source debug artifact: %w", err)
	}
	sourceInput := sourceInputMaterial(input.Path, input.RawInput)
	sessionID := resolvedSessionID(input.SessionID, doc.ID)
	output.Manifest.Metadata = manifestMetadataWithSessionID(output.Manifest.Metadata, sessionID)
	output.Manifest.SourceDigests = []string{doc.Digest}

	chunker := input.Prepared.chunker
	attachModuleManifestMetadata(&output, "chunker", chunker)
	var canonicalChunks []source.Chunk
	var chunkWarnings []contracts.Warning
	chunkCheckpoint, chunkDecision := checkpointLoader.Chunk(chunker.Key(), doc.Digest)
	recordCheckpointEvent(&output, checkpointLoader, string(StageChunk), "", chunker.Key(), chunkDecision)
	chunkStarted := time.Now().UTC()
	if err := writeDebugTimed(debugRecorder, "chunk/input.json", debugTimedEnvelope{
		Stage:     string(StageChunk),
		ModuleKey: chunker.Key(),
		StartedAt: chunkStarted,
		Payload: map[string]any{
			"reused":       chunkDecision.Reused,
			"decision":     chunkDecision,
			"source":       debugSourceDocumentEnvelope(doc),
			"source_input": debugContentEnvelope(sourceInput.Content, sourceInput.MediaType, nil, nil),
			"options":      redactSensitiveMap(input.pipeline.Chunk.Options),
			"metadata":     redactSensitiveMap(input.Metadata),
		},
	}); err != nil {
		return failOutput(output), fmt.Errorf("write chunk debug artifact: %w", err)
	}
	chunksAccepted := chunkDecision.Reused
	var chunkRejection *contracts.RejectedOutput
	if chunkDecision.Reused {
		canonicalChunks = cloneSourceChunks(chunkCheckpoint.Chunks)
		chunkWarnings = cloneWarnings(chunkCheckpoint.Warnings)
		output.Warnings = append(output.Warnings, chunkWarnings...)
	} else {
		if err := checkpoints.ChunkRunning(chunker.Key(), doc.Digest); err != nil {
			return failOutput(output), fmt.Errorf("write chunk checkpoint: %w", err)
		}
		chunksAccepted, chunkRejection, err = runWithRetry(ctx, input.pipeline.Chunk.Retries, func(attempt int) (bool, *contracts.RejectedOutput, error) {
			attemptStarted := time.Now().UTC()
			attemptPath := path.Join("chunk", fmt.Sprintf("attempt-%02d", attempt))
			attemptCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
			chunkResult, err := chunker.Chunk(attemptCtx, contracts.ChunkRequest{
				Source:      doc,
				SourceInput: sourceInput.Clone(),
				SessionID:   sessionID,
				References:  CloneReferenceSet(input.pipeline.ChunkReferences.ReferenceSet),
				LLMProfile:  input.pipeline.Chunk.LLMProfile,
				Metadata:    input.Metadata,
			})
			if err != nil {
				_ = writeDebugTimed(debugRecorder, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
					Stage:     string(StageChunk),
					ModuleKey: chunker.Key(),
					Attempt:   attempt,
					StartedAt: attemptStarted,
					Error:     err.Error(),
				}, llmScope))
				return false, nil, fmt.Errorf("chunk source with chunker %q: %w", chunker.Key(), err)
			}
			if len(chunkResult.Chunks) == 0 {
				err := fmt.Errorf("chunker %q returned no chunks", chunker.Key())
				_ = writeDebugTimed(debugRecorder, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
					Stage:     string(StageChunk),
					ModuleKey: chunker.Key(),
					Attempt:   attempt,
					StartedAt: attemptStarted,
					Error:     err.Error(),
				}, llmScope))
				return false, nil, err
			}
			chunks, err := validateAndCanonicalizeChunkResult(doc, chunkResult.Chunks)
			if err != nil {
				err := fmt.Errorf("validate chunks from chunker %q: %w", chunker.Key(), err)
				_ = writeDebugTimed(debugRecorder, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
					Stage:     string(StageChunk),
					ModuleKey: chunker.Key(),
					Attempt:   attempt,
					StartedAt: attemptStarted,
					Payload: map[string]any{
						"warnings": cloneWarnings(chunkResult.Warnings),
					},
					Error: err.Error(),
				}, llmScope))
				return false, nil, err
			}
			validationWarnings, rejection, err := r.validateChunks(attemptCtx, doc, chunker.Key(), chunks, sourceInput, sessionID, input.pipeline.ChunkReferences.ReferenceSet, input.Metadata, input.Prepared.chunkValidators, attempt, input.Debug)
			if err != nil || rejection != nil {
				_ = writeDebugTimed(debugRecorder, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
					Stage:     string(StageChunk),
					ModuleKey: chunker.Key(),
					Attempt:   attempt,
					StartedAt: attemptStarted,
					Payload: map[string]any{
						"chunks":    debugSourceChunkEnvelopes(chunks),
						"warnings":  append(cloneWarnings(chunkResult.Warnings), validationWarnings...),
						"rejection": debugRejectedOutputPtr(rejection),
					},
				}, llmScope))
				return false, rejection, err
			}
			canonicalChunks = chunks
			chunkWarnings = append(cloneWarnings(chunkResult.Warnings), validationWarnings...)
			if err := writeDebugTimed(debugRecorder, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{
				Stage:     string(StageChunk),
				ModuleKey: chunker.Key(),
				Attempt:   attempt,
				StartedAt: attemptStarted,
				Payload: map[string]any{
					"chunks":   debugSourceChunkEnvelopes(chunks),
					"warnings": chunkWarnings,
				},
			}, llmScope)); err != nil {
				return false, nil, err
			}
			return true, nil, nil
		})
		if err != nil {
			_ = checkpoints.ChunkFailed(chunker.Key(), doc.Digest, err)
			return failOutput(output), err
		}
		if !chunksAccepted {
			output.Rejected = append(output.Rejected, *chunkRejection)
			if err := checkpoints.ChunkRejected(chunker.Key(), doc.Digest, *chunkRejection); err != nil {
				return failOutput(output), fmt.Errorf("write chunk checkpoint: %w", err)
			}
		} else {
			output.Warnings = append(output.Warnings, chunkWarnings...)
			if err := checkpoints.ChunkSucceeded(chunker.Key(), doc.Digest, canonicalChunks, chunkWarnings); err != nil {
				return failOutput(output), fmt.Errorf("write chunk checkpoint: %w", err)
			}
		}
	}
	chunkDebugPayload := map[string]any{
		"reused":   chunkDecision.Reused,
		"accepted": chunksAccepted,
		"chunks":   debugSourceChunkEnvelopes(canonicalChunks),
		"warnings": chunkWarnings,
	}
	if chunkRejection != nil {
		chunkDebugPayload["rejection"] = debugRejectedOutputEnvelope(*chunkRejection)
	}
	if err := writeDebugTimed(debugRecorder, "chunk/output.json", debugTimedEnvelope{
		Stage:     string(StageChunk),
		ModuleKey: chunker.Key(),
		StartedAt: chunkStarted,
		Payload:   chunkDebugPayload,
	}); err != nil {
		return failOutput(output), fmt.Errorf("write chunk debug artifact: %w", err)
	}

	if chunksAccepted {
		laneOutput, laneErr := r.runLanes(ctx, input, checkpoints, checkpointLoader, doc, sourceInput, sessionID, canonicalChunks)
		mergeLaneOutput(&output, laneOutput)
		if laneErr != nil {
			return failOutput(output), laneErr
		}
	}

	if len(output.Rejected) > 0 {
		output.Manifest.ValidationStatus = "rejected"
	} else {
		output.Manifest.ValidationStatus = "approved"
	}
	populateOutputManifest(&output)
	output.Manifest.CompletedAt = timePtr(time.Now().UTC())

	encoder := input.Prepared.output
	attachModuleManifestMetadata(&output, "output", encoder)
	outputStarted := time.Now().UTC()
	if err := writeDebugTimed(debugRecorder, "output/input.json", debugTimedEnvelope{
		Stage:     string(StageOutput),
		ModuleKey: encoder.Key(),
		StartedAt: outputStarted,
		Payload: map[string]any{
			"manifest":          output.Manifest,
			"normalize_outputs": debugSerializedOutputEnvelopes(output.NormalizeOutputs),
			"rejected":          debugRejectedOutputEnvelopes(output.Rejected),
			"warnings":          output.Warnings,
			"options":           redactSensitiveMap(input.pipeline.Output.Options),
			"metadata":          redactSensitiveMap(input.Metadata),
		},
	}); err != nil {
		return failOutput(output), fmt.Errorf("write output debug artifact: %w", err)
	}
	encoded, err := encoder.Encode(ctx, contracts.OutputRequest{
		Manifest:         output.Manifest,
		NormalizeOutputs: cloneSerializedOutputs(output.NormalizeOutputs),
		Rejected:         cloneRejectedOutputs(output.Rejected),
		Warnings:         output.Warnings,
		LLMProfile:       input.pipeline.Output.LLMProfile,
		Metadata:         input.Metadata,
	})
	output.Warnings = append(output.Warnings, encoded.Warnings...)
	if err != nil {
		return failOutput(output), fmt.Errorf("encode output with encoder %q: %w", encoder.Key(), err)
	}
	files, err := outputFilesFromResult(encoded)
	if err != nil {
		return failOutput(output), fmt.Errorf("validate output files from encoder %q: %w", encoder.Key(), err)
	}
	output.OutputFiles = files
	if err := writeDebugTimed(debugRecorder, "output/output.json", debugTimedEnvelope{
		Stage:     string(StageOutput),
		ModuleKey: encoder.Key(),
		StartedAt: outputStarted,
		Payload: map[string]any{
			"files":    debugOutputFiles(files),
			"warnings": encoded.Warnings,
		},
	}); err != nil {
		return failOutput(output), fmt.Errorf("write output debug artifact: %w", err)
	}

	return output, nil
}

func runWithRetry(ctx context.Context, retries int, run func(attempt int) (bool, *contracts.RejectedOutput, error)) (bool, *contracts.RejectedOutput, error) {
	attempts := retries + 1
	var last *contracts.RejectedOutput
	for attempt := 1; attempt <= attempts; attempt++ {
		if err := ctx.Err(); err != nil {
			return false, nil, err
		}
		accepted, rejection, err := run(attempt)
		if err != nil {
			if attempt == attempts {
				return false, nil, fmt.Errorf("failed after %d attempt(s): %w", attempt, err)
			}
			if ctxErr := ctx.Err(); ctxErr != nil {
				return false, nil, ctxErr
			}
			continue
		}
		if accepted {
			return true, nil, nil
		}
		if rejection != nil {
			rejection.AttemptCount = attempt
			last = rejection
		}
		if ctxErr := ctx.Err(); ctxErr != nil {
			return false, nil, ctxErr
		}
		if attempt == attempts {
			if last == nil {
				last = &contracts.RejectedOutput{ReasonCode: "output_rejected", Message: "output rejected", AttemptCount: attempt}
			}
			return false, last, nil
		}
	}
	return false, last, nil
}

func (r *Runner) validateChunks(ctx context.Context, doc *source.SourceDocument, moduleKey string, chunks []source.Chunk, sourceInput contracts.LLMInputMaterial, sessionID string, references contracts.ReferenceSet, metadata map[string]any, prepared preparedValidatorChain, attempt int, debug DebugRecorder) ([]contracts.Warning, *contracts.RejectedOutput, error) {
	content, err := json.Marshal(chunks)
	if err != nil {
		return nil, nil, fmt.Errorf("encode canonical chunks for validation: %w", err)
	}
	schema := contracts.ArtifactSchema{ID: "notarius.source.chunks", Name: "notarius_source_chunks", Version: "v1", JSONSchema: []byte(`{"$schema":"https://json-schema.org/draft/2020-12/schema","type":"array"}`)}
	var warnings []contracts.Warning
	for index, item := range prepared.validators {
		binding := item.resolved.Binding
		started := time.Now().UTC()
		attemptPath := path.Join("validate", debugPathComponent(string(StageChunk)), "", debugPathComponent(moduleKey), fmt.Sprintf("%02d-%s-attempt-%02d", index+1, debugPathComponent(binding.Module), attempt))
		validatorCtx, llmScope := withDebugLLMScope(ctx, attemptPath)
		var result contracts.ValidationResult
		switch item.resolved.Target {
		case ValidatorTargetChunk:
			result, err = item.chunk.Validate(validatorCtx, contracts.ChunkValidationRequest{ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, Metadata: cloneMetadata(metadata), Chunks: cloneSourceChunks(chunks)})
		case ValidatorTargetSerialized:
			result, err = item.serialized.Validate(validatorCtx, contracts.SerializedValidationRequest{Stage: string(StageChunk), ModuleKey: moduleKey, Source: doc, SourceID: doc.ID, SourceInput: sourceInput.Clone(), SessionID: sessionID, References: CloneReferenceSet(references), LLMProfile: binding.LLMProfile, Metadata: cloneMetadata(metadata), Chunks: cloneSourceChunks(chunks), Schema: contracts.CloneArtifactSchema(schema), MediaType: "application/json", Content: append([]byte(nil), content...)})
		default:
			return nil, nil, fmt.Errorf("validator %q is incompatible with chunk validation", binding.Module)
		}
		debugContent := debugContentEnvelope(content, "application/json", nil, nil)
		debugContent.ContentDigest = debugContentDigest(content)
		debugCall := debugValidationCall{ValidatorName: binding.Module, Request: map[string]any{"stage": string(StageChunk), "module_key": moduleKey, "source_id": doc.ID, "schema": schema, "schema_digest": contracts.DigestArtifactSchema(schema), "content": debugContent, "metadata": redactSensitiveMap(metadata)}, Result: debugValidationResultEnvelope(result)}
		if err != nil {
			debugCall.Error = err.Error()
		}
		if debugErr := writeDebugTimed(debug, attemptPath+".json", debugEnvelopeWithLLMCalls(debugTimedEnvelope{Stage: string(StageChunk), ModuleKey: moduleKey, Attempt: attempt, StartedAt: started, Payload: debugCall, Error: debugCall.Error}, llmScope)); debugErr != nil {
			return nil, nil, debugErr
		}
		if err != nil {
			return nil, nil, fmt.Errorf("validate chunks with validator %q: %w", binding.Module, err)
		}
		if !result.Approved {
			reason := result.ReasonCode
			if reason == "" {
				reason = "output_rejected"
			}
			message := result.Message
			if message == "" {
				message = "output rejected"
			}
			return nil, &contracts.RejectedOutput{Stage: string(StageChunk), ModuleKey: moduleKey, ValidatorName: binding.Module, ReasonCode: reason, Message: message, AttemptCount: attempt, DiagnosticArtifactPath: result.DiagnosticArtifactPath}, nil
		}
		warnings = append(warnings, result.Warnings...)
	}
	return warnings, nil, nil
}

func resolvedValidatorChain(stage ModuleStage, laneID string, moduleKey string, chains []ResolvedValidatorChain) ResolvedValidatorChain {
	for _, chain := range chains {
		if chain.Stage != stage {
			continue
		}
		if chain.ModuleKey != moduleKey {
			continue
		}
		if strings.TrimSpace(chain.LaneID) != strings.TrimSpace(laneID) {
			continue
		}
		return ResolvedValidatorChain{
			Stage:      chain.Stage,
			LaneID:     chain.LaneID,
			ModuleKey:  chain.ModuleKey,
			Validators: cloneResolvedValidators(chain.Validators),
		}
	}
	return ResolvedValidatorChain{
		Stage:     stage,
		LaneID:    strings.TrimSpace(laneID),
		ModuleKey: strings.TrimSpace(moduleKey),
	}
}

func validateRunInput(input RunInput) error {
	if input.Prepared == nil {
		return fmt.Errorf("prepared pipeline must not be nil")
	}
	return validateResolvedPipeline(input.Prepared.resolved)
}

func validateResolvedPipeline(pipeline ResolvedPipeline) error {
	if pipeline.ID == "" {
		return fmt.Errorf("resolved pipeline id must not be empty")
	}
	if pipeline.Digest == "" {
		return fmt.Errorf("resolved pipeline digest must not be empty")
	}
	if pipeline.Input.Module == "" {
		return fmt.Errorf("resolved pipeline input module must not be empty")
	}
	if pipeline.Chunk.Module == "" {
		return fmt.Errorf("resolved pipeline chunk module must not be empty")
	}
	if pipeline.Output.Module == "" {
		return fmt.Errorf("resolved pipeline output module must not be empty")
	}
	if len(pipeline.ArtifactLanes) == 0 {
		return fmt.Errorf("resolved pipeline artifact lanes must not be empty")
	}
	for _, lane := range pipeline.ArtifactLanes {
		if lane.ID == "" {
			return fmt.Errorf("resolved pipeline artifact lane id must not be empty")
		}
		if lane.Extract.Module == "" {
			return fmt.Errorf("resolved pipeline lane %q extract module must not be empty", lane.ID)
		}
		if lane.Merge.Module == "" {
			return fmt.Errorf("resolved pipeline lane %q merge module must not be empty", lane.ID)
		}
		if lane.Normalize.Module == "" {
			return fmt.Errorf("resolved pipeline lane %q normalize module must not be empty", lane.ID)
		}
		if len(lane.Validators) > 0 {
			return fmt.Errorf("resolved pipeline lane %q validators are not supported at artifact lane level; use extract.validators, merge.validators, or normalize.validators", lane.ID)
		}
	}
	return nil
}

func startedTime(t time.Time) time.Time {
	if t.IsZero() {
		return time.Now().UTC()
	}
	return t.UTC()
}

func manifestFromPipeline(input RunInput) artifacts.RunManifest {
	startedAt := input.StartedAt
	if startedAt.IsZero() {
		startedAt = time.Now().UTC()
	}
	runID := strings.TrimSpace(input.RunID)
	if runID == "" {
		runID = fmt.Sprintf("run-%d", startedAt.UnixNano())
	}

	pipeline := input.pipeline
	manifest := artifacts.RunManifest{
		PipelineID:      pipeline.ID,
		PipelineDigest:  pipeline.Digest,
		InputModule:     pipeline.Input.Module,
		Chunker:         pipeline.Chunk.Module,
		OutputEncoder:   pipeline.Output.Module,
		ArtifactLanes:   make([]artifacts.ArtifactLaneManifest, 0, len(pipeline.ArtifactLanes)),
		ValidatorChains: validatorChainManifests(pipeline.ValidatorChains),
		RunID:           runID,
		StartedAt:       timePtr(startedAt),
		References:      ReferenceProvenance(pipeline),
		LLMProfiles:     cloneLLMProfiles(input.LLMProfiles),
	}
	// The runner does not currently maintain a cache or idempotency key. Reference
	// digests are recorded in manifest provenance and intentionally kept separate
	// from source_digests.

	for _, lane := range pipeline.ArtifactLanes {
		laneManifest := artifacts.ArtifactLaneManifest{
			ID:         lane.ID,
			Extractor:  lane.Extract.Module,
			Merger:     lane.Merge.Module,
			Normalizer: lane.Normalize.Module,
		}
		manifest.ArtifactLanes = append(manifest.ArtifactLanes, laneManifest)
	}
	return manifest
}

func validatorChainManifests(chains []ResolvedValidatorChain) []artifacts.ValidatorChainManifest {
	if len(chains) == 0 {
		return nil
	}
	manifests := make([]artifacts.ValidatorChainManifest, 0, len(chains))
	for _, chain := range chains {
		manifest := artifacts.ValidatorChainManifest{
			Stage:      string(chain.Stage),
			LaneID:     chain.LaneID,
			ModuleKey:  chain.ModuleKey,
			Validators: make([]artifacts.ValidatorManifest, 0, len(chain.Validators)),
		}
		for _, validator := range chain.Validators {
			manifest.Validators = append(manifest.Validators, artifacts.ValidatorManifest{
				Key:            validator.Binding.Module,
				ExecutionClass: string(validator.ExecutionClass),
			})
		}
		manifests = append(manifests, manifest)
	}
	return manifests
}

func failOutput(output RunOutput) RunOutput {
	if output.Manifest.PipelineID != "" {
		populateOutputManifest(&output)
		output.Manifest.ValidationStatus = "failed"
		output.Manifest.CompletedAt = timePtr(time.Now().UTC())
	}
	return output
}

func recordCheckpointEvent(output *RunOutput, loader CheckpointLoader, stage string, laneID string, moduleKey string, decision CheckpointDecision) {
	if output == nil || loader == nil || !loader.Enabled() {
		return
	}
	action := "executed"
	if decision.Reused {
		action = "reused"
	}
	output.CheckpointEvents = append(output.CheckpointEvents, CheckpointEvent{
		Stage:     stage,
		LaneID:    laneID,
		ModuleKey: moduleKey,
		Action:    action,
		Reason:    decision.Reason,
	})
}

func populateOutputManifest(output *RunOutput) {
	if output == nil {
		return
	}
	output.Manifest.NormalizedOutputs = normalizedOutputManifests(output.NormalizeOutputs)
	output.Manifest.RejectedOutputs = rejectedOutputManifests(output.Rejected)
}

func normalizedOutputManifests(outputs []contracts.SerializedOutput) []artifacts.NormalizedOutputManifest {
	if len(outputs) == 0 {
		return nil
	}
	manifests := make([]artifacts.NormalizedOutputManifest, 0, len(outputs))
	for _, output := range outputs {
		manifests = append(manifests, artifacts.NormalizedOutputManifest{
			LaneID:    output.LaneID,
			ModuleKey: output.NormalizerKey,
			SourceID:  output.SourceID,
			MediaType: output.Artifact.MediaType,
			Schema: artifacts.OutputSchemaProvenance{
				ID:      output.Artifact.Schema.ID,
				Name:    output.Artifact.Schema.Name,
				Version: output.Artifact.Schema.Version,
			},
		})
	}
	return manifests
}

func rejectedOutputManifests(rejected []contracts.RejectedOutput) []artifacts.RejectedOutputManifest {
	if len(rejected) == 0 {
		return nil
	}
	manifests := make([]artifacts.RejectedOutputManifest, 0, len(rejected))
	for _, output := range rejected {
		manifests = append(manifests, artifacts.RejectedOutputManifest{
			Stage:                  output.Stage,
			LaneID:                 output.LaneID,
			ModuleKey:              output.ModuleKey,
			ChunkID:                output.ChunkID,
			ChunkIndex:             output.ChunkIndex,
			ValidatorName:          output.ValidatorName,
			ReasonCode:             output.ReasonCode,
			Message:                output.Message,
			AttemptCount:           output.AttemptCount,
			DiagnosticArtifactPath: output.DiagnosticArtifactPath,
		})
	}
	return manifests
}

func attachModuleManifestMetadata(output *RunOutput, moduleKey string, module any) {
	if output == nil {
		return
	}
	metadata, ok := moduleManifestMetadata(module)
	if !ok {
		return
	}
	if output.Manifest.ModuleMetadata == nil {
		output.Manifest.ModuleMetadata = make(map[string]map[string]any)
	}
	output.Manifest.ModuleMetadata[moduleKey] = metadata
}

func moduleManifestMetadata(module any) (map[string]any, bool) {
	provider, ok := module.(contracts.ManifestMetadataProvider)
	if !ok {
		return nil, false
	}

	moduleMetadata := cloneMetadata(provider.ManifestMetadata())
	if len(moduleMetadata) == 0 {
		return nil, false
	}
	return moduleMetadata, true
}

func outputFilesFromResult(result contracts.OutputResult) ([]contracts.OutputFile, error) {
	out := make([]contracts.OutputFile, 0, len(result.Files))
	for _, file := range result.Files {
		if err := validateOutputFileName(file.Name); err != nil {
			return nil, err
		}
		out = append(out, contracts.OutputFile{
			Name:        file.Name,
			ContentType: file.ContentType,
			Bytes:       append([]byte(nil), file.Bytes...),
		})
	}
	return out, nil
}

func validateOutputFileName(name string) error {
	if strings.TrimSpace(name) == "" {
		return fmt.Errorf("output file name must not be empty")
	}
	if strings.Contains(name, "\\") {
		return fmt.Errorf("output file name %q must use slash-separated relative paths", name)
	}
	if path.IsAbs(name) {
		return fmt.Errorf("output file name %q must be relative", name)
	}
	if strings.Contains(name, "..") {
		return fmt.Errorf("output file name %q must not contain ..", name)
	}
	cleaned := path.Clean(name)
	if cleaned == "." || cleaned != name {
		return fmt.Errorf("output file name %q must be clean", name)
	}
	return nil
}

func cloneMetadata(metadata map[string]any) map[string]any {
	if len(metadata) == 0 {
		return nil
	}
	out := make(map[string]any, len(metadata))
	for key, value := range metadata {
		out[key] = value
	}
	return out
}

func cloneLLMProfiles(profiles []artifacts.LLMProfileManifest) []artifacts.LLMProfileManifest {
	if len(profiles) == 0 {
		return nil
	}
	return append([]artifacts.LLMProfileManifest(nil), profiles...)
}

func llmProfileManifests(client contracts.StructuredLLMClient) []artifacts.LLMProfileManifest {
	provider, ok := client.(contracts.LLMProfileManifestProvider)
	if !ok {
		return nil
	}
	return provider.LLMProfileManifests()
}

func mergeLLMProfileManifests(sources ...[]artifacts.LLMProfileManifest) []artifacts.LLMProfileManifest {
	merged := make(map[string]artifacts.LLMProfileManifest)
	for _, source := range sources {
		for _, profile := range source {
			id := strings.TrimSpace(profile.ID)
			provider := strings.TrimSpace(profile.Provider)
			model := strings.TrimSpace(profile.Model)
			key := id + "\x00" + provider + "\x00" + model
			if _, exists := merged[key]; exists {
				continue
			}
			merged[key] = artifacts.LLMProfileManifest{
				ID:       id,
				Provider: provider,
				Model:    model,
			}
		}
	}
	if len(merged) == 0 {
		return nil
	}
	keys := make([]string, 0, len(merged))
	for key := range merged {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	out := make([]artifacts.LLMProfileManifest, 0, len(keys))
	for _, key := range keys {
		out = append(out, merged[key])
	}
	return out
}

func sourceInputMaterial(inputPath string, content []byte) contracts.LLMInputMaterial {
	return contracts.NewLLMInputMaterial(
		"source",
		sourceInputMediaType(inputPath),
		content,
		sourceInputDigest(content),
		sourceInputOriginURI(inputPath),
	)
}

func chunkInputMaterial(sourceInput contracts.LLMInputMaterial, chunk source.Chunk) contracts.LLMInputMaterial {
	return contracts.NewLLMInputMaterial(
		"source",
		chunk.MediaType,
		chunk.Content,
		sourceInputDigest(chunk.Content),
		sourceInput.OriginURI,
	)
}

func sourceInputMediaType(inputPath string) string {
	extension := strings.ToLower(filepath.Ext(strings.TrimSpace(inputPath)))
	if extension == ".json" {
		return "application/json"
	}
	mediaType := mime.TypeByExtension(extension)
	if strings.TrimSpace(mediaType) == "" {
		return unknownMediaType
	}
	return canonicalMediaType(mediaType)
}

func sourceInputDigest(content []byte) string {
	sum := sha256.Sum256(content)
	return "sha256:" + hex.EncodeToString(sum[:])
}

func sourceInputOriginURI(inputPath string) string {
	if strings.TrimSpace(inputPath) == "" {
		return ""
	}
	return fileURI(inputPath)
}

func resolvedSessionID(explicit string, sourceDocumentID string) string {
	if trimmed := strings.TrimSpace(explicit); trimmed != "" {
		return trimmed
	}
	return strings.TrimSpace(sourceDocumentID)
}

func manifestMetadataWithSessionID(metadata map[string]any, sessionID string) map[string]any {
	out := cloneMetadata(metadata)
	if strings.TrimSpace(sessionID) == "" {
		return out
	}
	if out == nil {
		out = make(map[string]any)
	}
	out["session_id"] = sessionID
	return out
}

func cloneWarnings(warnings []contracts.Warning) []contracts.Warning {
	if len(warnings) == 0 {
		return nil
	}
	return append([]contracts.Warning(nil), warnings...)
}

func cloneSourceChunkPtr(chunk *source.Chunk) *source.Chunk {
	if chunk == nil {
		return nil
	}
	cloned := cloneSourceChunk(*chunk)
	return &cloned
}

func cloneSourceChunk(chunk source.Chunk) source.Chunk {
	chunk.Content = append([]byte(nil), chunk.Content...)
	chunk.Units = cloneSourceUnits(chunk.Units)
	chunk.Metadata = cloneMetadata(chunk.Metadata)
	return chunk
}

func cloneSourceChunks(chunks []source.Chunk) []source.Chunk {
	if len(chunks) == 0 {
		return nil
	}
	out := make([]source.Chunk, 0, len(chunks))
	for _, chunk := range chunks {
		out = append(out, cloneSourceChunk(chunk))
	}
	return out
}

func cloneSourceUnits(units []source.SourceUnit) []source.SourceUnit {
	if len(units) == 0 {
		return nil
	}
	out := make([]source.SourceUnit, 0, len(units))
	for _, unit := range units {
		out = append(out, cloneSourceUnit(unit))
	}
	return out
}

func cloneSerializedOutputs(outputs []contracts.SerializedOutput) []contracts.SerializedOutput {
	if len(outputs) == 0 {
		return nil
	}
	out := make([]contracts.SerializedOutput, 0, len(outputs))
	for _, output := range outputs {
		out = append(out, contracts.CloneSerializedOutput(output))
	}
	return out
}

func cloneRejectedOutputs(rejected []contracts.RejectedOutput) []contracts.RejectedOutput {
	if len(rejected) == 0 {
		return nil
	}
	return append([]contracts.RejectedOutput(nil), rejected...)
}

func timePtr(t time.Time) *time.Time {
	return &t
}