notarius/internal/cli/run.go

package cli

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"sort"
	"strings"
	"time"

	"gitea.maximumdirect.net/eric/notarius/internal/core/artifacts"
	"gitea.maximumdirect.net/eric/notarius/internal/core/config"
	"gitea.maximumdirect.net/eric/notarius/internal/core/debugbundle"
	"gitea.maximumdirect.net/eric/notarius/internal/core/fileio"
	"gitea.maximumdirect.net/eric/notarius/internal/framework/checkpoint"
	"gitea.maximumdirect.net/eric/notarius/internal/framework/chunkplan"
	"gitea.maximumdirect.net/eric/notarius/internal/framework/contracts"
	frameworkdebug "gitea.maximumdirect.net/eric/notarius/internal/framework/debug"
	frameworkllm "gitea.maximumdirect.net/eric/notarius/internal/framework/llm"
	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)

const defaultConfigPath = "/usr/local/etc/notarius/config.yml"
const usage = `Usage:
  notarius help
  notarius run <pipeline-id> --input path/to/source.json [--json] [flags]
  notarius config validate --config path/to/config.yml [--pipeline pipeline-id] [--only lane-a,lane-b]
  notarius pipelines list --config path/to/config.yml [--json]
`

type Options struct {
	Catalog               pipeline.ModuleCatalog
	Registries            pipeline.Registries
	LLMClientFactory      LLMClientFactory
	RunIDGenerator        RunIDGenerator
	LookupEnv             func(string) (string, bool)
	Now                   func() time.Time
	UserCacheDir          func() (string, error)
	ChunkPlanStoreFactory pipeline.ChunkPlanStoreFactory
	DebugRecorderFactory  func(string) (pipeline.DebugRecorder, error)
	DebugTerminalFactory  func(*debugbundle.SummaryWriter) DebugTerminalWriter
	promptKitAssets       *frameworkllm.AssetRegistry
}

type LLMRuntimeOverrides struct {
	ReasoningEffort *string
}

type LLMClientFactory func(ctx context.Context, cfg config.Config, profileID string, overrides LLMRuntimeOverrides) (contracts.StructuredLLMClient, []artifacts.LLMProfileManifest, error)

// Run executes the command-line interface and returns a process exit code.
func Run(args []string, stdout, stderr io.Writer) int {
	return RunWithOptions(args, stdout, stderr, Options{})
}

func RunWithOptions(args []string, stdout, stderr io.Writer, opts Options) int {
	var err error
	opts, err = normalizeOptions(opts)
	if err != nil {
		fmt.Fprintf(stderr, "notarius: %v\n", err)
		return 1
	}
	if len(args) == 0 {
		writeUsage(stdout)
		return 0
	}

	switch args[0] {
	case "help", "--help", "-h":
		writeUsage(stdout)
		return 0
	case "config":
		return runConfig(args[1:], stdout, stderr, opts)
	case "pipelines":
		return runPipelines(args[1:], stdout, stderr, opts)
	case "run":
		return runPipelineCommand(args[1:], stdout, stderr, opts)
	default:
		fmt.Fprintf(stderr, "notarius: unknown command %q\n", args[0])
		writeUsage(stderr)
		return 2
	}
}

func writeUsage(w io.Writer) {
	fmt.Fprint(w, usage)
}

func normalizeOptions(opts Options) (Options, error) {
	if opts.LookupEnv == nil {
		opts.LookupEnv = os.LookupEnv
	}
	if opts.Now == nil {
		opts.Now = time.Now
	}
	if opts.RunIDGenerator == nil {
		opts.RunIDGenerator = defaultRunIDGenerator
	}
	if opts.UserCacheDir == nil {
		opts.UserCacheDir = os.UserCacheDir
	}
	if opts.ChunkPlanStoreFactory == nil {
		opts.ChunkPlanStoreFactory = chunkplan.NewFilesystemStore
	}
	if opts.DebugRecorderFactory == nil {
		opts.DebugRecorderFactory = frameworkdebug.NewFilesystemRecorder
	}
	if opts.DebugTerminalFactory == nil {
		opts.DebugTerminalFactory = func(writer *debugbundle.SummaryWriter) DebugTerminalWriter { return writer }
	}
	if isEmptyCatalog(opts.Catalog) && isEmptyRegistries(opts.Registries) {
		components, err := newProductionComponents()
		if err != nil {
			return Options{}, err
		}
		opts.Registries = components.registries
		opts.Catalog = catalogFromRegistries(components.registries)
		opts.promptKitAssets = components.assets
	}
	if opts.LLMClientFactory == nil {
		if opts.promptKitAssets == nil {
			assets, err := productionPromptAssets()
			if err != nil {
				return Options{}, err
			}
			opts.promptKitAssets = assets
		}
		opts.LLMClientFactory = productionLLMClientFactoryWithAssets(opts.promptKitAssets)
	}
	return opts, nil
}

func runPipelineCommand(args []string, stdout, stderr io.Writer, opts Options) int {
	fs := flag.NewFlagSet("run", flag.ContinueOnError)
	fs.SetOutput(io.Discard)
	configPath := fs.String("config", "", "config file path")
	inputPath := fs.String("input", "", "source input file path")
	onlyRaw := fs.String("only", "", "comma-separated artifact lanes")
	outputDir := fs.String("output-dir", "", "output directory")
	machineOutput := fs.Bool("json", false, "write the successful run result as JSON")
	debug := fs.Bool("debug", false, "write a debug bundle")
	debugDir := fs.String("debug-dir", "", "debug bundle directory")
	llmProfile := singleValueFlag{name: "--llm-profile"}
	reasoningEffort := singleValueFlag{name: "--reasoning-effort"}
	clearReasoningEffort := fs.Bool("clear-reasoning-effort", false, "clear the LLM profile reasoning effort")
	resume := fs.Bool("resume", false, "reuse compatible recorded checkpoints")
	recomputeStep := singleValueFlag{name: "--recompute-step"}
	chunkCache := chunkCacheFlag{}
	requestedSessionID := sessionIDFlag{}
	referenceFlags := stringListFlag{}
	withoutReferenceFlags := stringListFlag{}
	fs.Var(&requestedSessionID, "session-id", "prompt session identifier")
	fs.Var(&llmProfile, "llm-profile", "LLM profile override")
	fs.Var(&reasoningEffort, "reasoning-effort", "reasoning effort override")
	fs.Var(&chunkCache, "chunk_cache", "chunk plan cache mode: auto, bypass, or refresh")
	fs.Var(&referenceFlags, "reference", "reference binding, as slot=path, chunk.slot=path, merge.slot=path, lane.slot=path, lane.extract.slot=path, lane.merge.slot=path, or lane.normalize.slot=path")
	fs.Var(&withoutReferenceFlags, "without-reference", "unbind a reference, using the same selector forms as --reference")
	fs.Var(&recomputeStep, "recompute-step", "recompute one ordered pipeline step and dependent lanes")
	if err := validateRunFlagValues(args); err != nil {
		fmt.Fprintf(stderr, "notarius: %v\n", err)
		return 2
	}
	if err := fs.Parse(reorderRunArgs(args)); err != nil {
		fmt.Fprintf(stderr, "notarius: %v\n", err)
		return 2
	}
	if fs.NArg() == 0 {
		fmt.Fprintln(stderr, "notarius: run requires a pipeline ID")
		return 2
	}
	if fs.NArg() > 1 {
		fmt.Fprintf(stderr, "notarius: unexpected argument %q\n", fs.Arg(1))
		return 2
	}
	pipelineID := strings.TrimSpace(fs.Arg(0))
	if pipelineID == "" {
		fmt.Fprintln(stderr, "notarius: run requires a pipeline ID")
		return 2
	}
	if strings.TrimSpace(*inputPath) == "" {
		fmt.Fprintln(stderr, "notarius: run requires --input")
		return 2
	}
	if strings.TrimSpace(*debugDir) != "" && !*debug {
		fmt.Fprintln(stderr, "notarius: --debug-dir requires --debug")
		return 2
	}
	if strings.TrimSpace(*outputDir) == "" && flagWasProvided(args, "--output-dir") {
		fmt.Fprintln(stderr, "notarius: --output-dir must not be empty")
		return 2
	}
	if strings.TrimSpace(*debugDir) == "" && flagWasProvided(args, "--debug-dir") {
		fmt.Fprintln(stderr, "notarius: --debug-dir must not be empty")
		return 2
	}
	if requestedSessionID.set && strings.TrimSpace(requestedSessionID.value) == "" {
		fmt.Fprintln(stderr, "notarius: --session-id must not be empty")
		return 2
	}
	if llmProfile.set && strings.TrimSpace(llmProfile.value) == "" {
		fmt.Fprintln(stderr, "notarius: --llm-profile must not be empty")
		return 2
	}
	if reasoningEffort.set && *clearReasoningEffort {
		fmt.Fprintln(stderr, "notarius: --reasoning-effort cannot be combined with --clear-reasoning-effort")
		return 2
	}
	if reasoningEffort.set && strings.TrimSpace(reasoningEffort.value) == "" {
		fmt.Fprintln(stderr, "notarius: --reasoning-effort must not be empty")
		return 2
	}
	runtimeOverrides := LLMRuntimeOverrides{}
	if reasoningEffort.set {
		value := strings.TrimSpace(reasoningEffort.value)
		runtimeOverrides.ReasoningEffort = &value
	} else if *clearReasoningEffort {
		value := ""
		runtimeOverrides.ReasoningEffort = &value
	}
	only, err := parseOnly(*onlyRaw)
	if err != nil {
		fmt.Fprintf(stderr, "notarius: %v\n", err)
		return 2
	}
	if recomputeStep.set && strings.TrimSpace(recomputeStep.value) == "" {
		fmt.Fprintln(stderr, "notarius: --recompute-step must not be empty")
		return 2
	}
	if recomputeStep.set && len(only) > 0 {
		fmt.Fprintln(stderr, "notarius: --recompute-step cannot be combined with --only")
		return 2
	}
	referenceRequests, err := parseReferenceFlags(referenceFlags)
	if err != nil {
		fmt.Fprintf(stderr, "notarius: %v\n", err)
		return 2
	}
	referenceUnbindRequests, err := parseReferenceUnbindFlags(withoutReferenceFlags)
	if err != nil {
		fmt.Fprintf(stderr, "notarius: %v\n", err)
		return 2
	}

	cfg, loadedConfigPath, err := loadConfig(*configPath, opts)
	if err != nil {
		fmt.Fprintf(stderr, "notarius: %v\n", err)
		return 1
	}
	if chunkCache.set {
		cfg.Cache.ChunkPlans.Mode = chunkCache.value
	}
	if dir := strings.TrimSpace(*outputDir); dir != "" {
		cfg.Output.Directory = dir
	}
	if dir := strings.TrimSpace(*debugDir); dir != "" {
		cfg.Debug.Directory = dir
	}
	if err := cfg.Validate(); err != nil {
		fmt.Fprintf(stderr, "notarius: %v\n", err)
		return 1
	}
	if *resume && !cfg.Cache.Checkpoints.Enabled {
		fmt.Fprintln(stderr, "notarius: --resume requires cache.checkpoints.enabled: true")
		return 1
	}
	if recomputeStep.set && !*resume {
		fmt.Fprintln(stderr, "notarius: --recompute-step requires --resume")
		return 2
	}
	if recomputeStep.set && !cfg.Cache.Checkpoints.Enabled {
		fmt.Fprintln(stderr, "notarius: --recompute-step requires cache.checkpoints.enabled: true")
		return 1
	}

	startedAt := opts.Now().UTC()
	runID, err := opts.RunIDGenerator(startedAt)
	if err != nil {
		fmt.Fprintf(stderr, "notarius: generate run ID: %v\n", err)
		return 1
	}
	if err := validateRunID(runID); err != nil {
		fmt.Fprintf(stderr, "notarius: invalid generated run ID: %v\n", err)
		return 1
	}
	runOutputDir := filepath.Join(cfg.Output.Directory, runID)
	commandState := newPipelineCommandState(runID, pipelineID, runOutputDir)
	var summary *debugbundle.SummaryWriter
	var terminalWriter DebugTerminalWriter
	debugPath := ""
	debugRecorder := pipeline.NoopDebugRecorder()
	if *debug {
		bundle, err := debugbundle.Allocate(cfg.Debug.Directory, runID, startedAt)
		if err != nil {
			fmt.Fprintf(stderr, "notarius: %v\n", err)
			return 1
		}
		debugPath, summary = bundle.Path(), bundle.Summary()
		commandState.setDebugPath(debugPath)
		terminalWriter = opts.DebugTerminalFactory(summary)
		if terminalWriter == nil {
			terminalWriter = summary
		}
		debugRecorder, err = opts.DebugRecorderFactory(bundle.TraceRoot())
		if err != nil {
			return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("create debug recorder: %w", err))
		}
		debugRecorder = pipeline.SynchronizedDebugRecorder(debugRecorder)
	}
	invocation := debugbundle.Invocation{
		Operation:               "run",
		PipelineID:              pipelineID,
		InputPath:               strings.TrimSpace(*inputPath),
		ConfigPath:              loadedConfigPath,
		ConfigSource:            configSource(*configPath),
		OnlyLanes:               append([]string(nil), only...),
		ChunkCacheOverride:      chunkCache.explicitValue(),
		ReasoningEffortOverride: runtimeOverrides.ReasoningEffort,
		Resume:                  *resume,
		RecomputeStep:           strings.TrimSpace(recomputeStep.value),
		RunID:                   runID,
		StartedAt:               startedAt,
	}
	if err := writeSummary(summary, func() error { return summary.WriteInvocation(invocation) }); err != nil {
		return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug invocation metadata: %w", err))
	}

	catalog, err := effectiveCatalog(opts)
	if err != nil {
		return failPipelineCommand(stderr, commandState, terminalWriter, err)
	}
	referenceOverrides, referenceUnbinds, err := resolveCLIReferenceRequests(cfg, pipelineID, only, catalog, referenceRequests, referenceUnbindRequests)
	if err != nil {
		return failPipelineCommand(stderr, commandState, terminalWriter, err)
	}
	effective, err := cfg.Resolve(config.ResolveInput{
		PipelineID:         pipelineID,
		Only:               only,
		Catalog:            catalog,
		LLMProfileOverride: strings.TrimSpace(llmProfile.value),
		ReferenceOverrides: referenceOverrides,
		ReferenceUnbinds:   referenceUnbinds,
	})
	if err != nil {
		return failPipelineCommand(stderr, commandState, terminalWriter, err)
	}
	profileIDs := effectiveLLMProfileIDs(effective.ResolvedPipeline)
	if err := validateExplicitPromptKitProfiles(context.Background(), effective.Config, profileIDs, opts.promptKitAssets); err != nil {
		return failPipelineCommand(stderr, commandState, terminalWriter, err)
	}
	workingDir, err := os.Getwd()
	if err != nil {
		return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("resolve working directory: %w", err))
	}
	materialized, referenceWarnings, err := pipeline.MaterializeReferences(effective.ResolvedPipeline, catalog, pipeline.ReferenceMaterializationOptions{
		ConfigPath: loadedConfigPath,
		WorkingDir: workingDir,
	})
	if err != nil {
		return failPipelineCommand(stderr, commandState, terminalWriter, err)
	}
	effective.ResolvedPipeline = materialized
	checkpointPolicy := pipeline.CheckpointExecutionPolicy{}
	if recomputeStep.set {
		checkpointPolicy, err = recomputePolicy(effective.ResolvedPipeline, recomputeStep.value)
		if err != nil {
			return failPipelineCommand(stderr, commandState, terminalWriter, err)
		}
	}
	invocation.PipelineDigest = effective.ResolvedPipeline.Digest
	if err := writeSummary(summary, func() error { return summary.WriteInvocation(invocation) }); err != nil {
		return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug invocation metadata: %w", err))
	}
	if err := writeSummary(summary, func() error { return summary.WriteRedactedEffectiveConfig(effective) }); err != nil {
		return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug effective config: %w", err))
	}
	if err := writeSummary(summary, func() error { return summary.WriteResolvedPipeline(effective) }); err != nil {
		return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug resolved pipeline: %w", err))
	}
	if err := writeSummary(summary, func() error {
		return summary.WriteResolvedReferences(pipeline.ReferenceProvenance(effective.ResolvedPipeline))
	}); err != nil {
		return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug resolved references: %w", err))
	}

	registries, err := effectiveRegistries(opts)
	if err != nil {
		return failPipelineCommand(stderr, commandState, terminalWriter, err)
	}

	ctx := context.Background()
	factoryProfileID := ""
	if len(profileIDs) == 1 {
		factoryProfileID = profileIDs[0]
	}
	llmClient, llmProfiles, err := opts.LLMClientFactory(ctx, effective.Config, factoryProfileID, runtimeOverrides)
	if err != nil {
		return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("create LLM client for profile %q: %w", factoryProfileID, err))
	}
	var llmFingerprints []checkpoint.Fingerprint
	if effective.Config.Cache.Checkpoints.Enabled {
		llmFingerprints, err = llmCheckpointFingerprints(llmClient)
		if err != nil {
			return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("prepare LLM checkpoint identity: %w", err))
		}
	}
	llmClient = pipeline.WithDebugLLMRecording(llmClient, debugRecorder)
	prepared, err := pipeline.Prepare(effective.ResolvedPipeline, registries, pipeline.ModuleDependencies{LLM: llmClient})
	if err != nil {
		return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("prepare pipeline %q: %w", pipelineID, err))
	}
	rawInput, err := os.ReadFile(strings.TrimSpace(*inputPath))
	if err != nil {
		return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("read input %q: %w", strings.TrimSpace(*inputPath), err))
	}
	effectiveSessionID, err := resolvePromptSessionID(requestedSessionID.value, effective.ResolvedPipeline.Input.Module, rawInput)
	if err != nil {
		return failPipelineCommand(stderr, commandState, terminalWriter, err)
	}
	invocation.SessionID = effectiveSessionID
	if err := writeSummary(summary, func() error { return summary.WriteInvocation(invocation) }); err != nil {
		return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug invocation metadata: %w", err))
	}
	chunkPlans, err := chunkPlanStoreForRun(effective.Config.Cache.ChunkPlans, opts)
	if err != nil {
		return failPipelineCommand(stderr, commandState, terminalWriter, err)
	}
	checkpointRecorder, checkpointLoader, err := checkpointHandlersForRun(effective.Config.Cache.Checkpoints, opts, effective.ResolvedPipeline, prepared.CheckpointFingerprints(), llmFingerprints, rawInput, only, llmProfiles, strings.TrimSpace(llmProfile.value), effectiveSessionID, runtimeOverrides, *resume)
	if err != nil {
		return failPipelineCommand(stderr, commandState, terminalWriter, err)
	}

	output, err := pipeline.New().Run(ctx, pipeline.RunInput{
		Prepared:         prepared,
		Path:             strings.TrimSpace(*inputPath),
		RawInput:         rawInput,
		SessionID:        effectiveSessionID,
		RunID:            runID,
		StartedAt:        startedAt,
		LLMProfiles:      llmProfiles,
		Metadata:         runMetadata(effective.Config.Output.Directory, debugPath),
		Warnings:         referenceWarnings,
		ChunkCacheMode:   effective.Config.Cache.ChunkPlans.Mode,
		ChunkPlans:       chunkPlans,
		Checkpoints:      checkpointRecorder,
		Checkpoint:       checkpointLoader,
		CheckpointPolicy: checkpointPolicy,
		Debug:            debugRecorder,
		ExtractWorkers:   cfg.Concurrency.StageWorkers["extract"],
	})
	commandState.observeOutput(output)
	if err != nil {
		primaryErr := fmt.Errorf("run pipeline %q: %w", pipelineID, err)
		if output.Manifest.PipelineID != "" {
			if summaryErr := writePartialSummary(summary, output); summaryErr != nil {
				return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr, fmt.Errorf("write debug summary: %w", summaryErr))
			}
		}
		return failPipelineCommand(stderr, commandState, terminalWriter, primaryErr)
	}

	if err := writePartialSummary(summary, output); err != nil {
		return failPipelineCommand(stderr, commandState, terminalWriter, fmt.Errorf("write debug summary: %w", err))
	}
	var encodedResult []byte
	if *machineOutput {
		result, err := newRunResult(effective.ResolvedPipeline, output, runOutputDir, debugPath)
		if err != nil {
			return failPipelineCommand(stderr, commandState, terminalWriter, err)
		}
		encodedResult, err = encodeRunResult(result)
		if err != nil {
			return failPipelineCommand(stderr, commandState, terminalWriter, err)
		}
	}
	if err := writeOutputFiles(runOutputDir, output.OutputFiles); err != nil {
		return failPipelineCommand(stderr, commandState, terminalWriter, err)
	}
	if primaryErr, persistenceErr := commandState.terminalize(terminalWriter, nil); primaryErr != nil {
		return writePipelineCommandFailure(stderr, commandState, primaryErr, persistenceErr)
	}

	if *machineOutput {
		if err := writeRunResult(stdout, encodedResult); err != nil {
			return writePipelineCommandFailure(stderr, commandState, errors.New("write run result"), nil)
		}
	} else {
		fmt.Fprintf(stdout, "pipeline %q complete: outputs=%d rejected=%d output=%s\n", effective.PipelineID, len(output.NormalizeOutputs), len(output.Rejected), runOutputDir)
		if debugPath != "" {
			fmt.Fprintf(stdout, "debug=%s\n", debugPath)
		}
	}
	if len(output.Warnings) > 0 {
		fmt.Fprintf(stderr, "notarius: run completed with %d warning(s)\n", len(output.Warnings))
	}
	return 0
}

func writeSummary(summary *debugbundle.SummaryWriter, write func() error) error {
	if summary == nil {
		return nil
	}
	return write()
}

func writePartialSummary(summary *debugbundle.SummaryWriter, output pipeline.RunOutput) error {
	if summary == nil {
		return nil
	}
	if err := summary.WriteRunManifest(output.Manifest); err != nil {
		return err
	}
	if output.ChunkPlan != nil {
		if err := summary.WriteChunkPlan(*output.ChunkPlan); err != nil {
			return err
		}
	}
	if err := summary.WriteWarnings(output.Warnings); err != nil {
		return err
	}
	return summary.WriteCheckpointEvents(output.CheckpointEvents)
}

func checkpointHandlersForRun(
	settings config.CheckpointCacheConfig,
	opts Options,
	resolved pipeline.ResolvedPipeline,
	componentFingerprints []pipeline.CheckpointFingerprint,
	llmFingerprints []checkpoint.Fingerprint,
	rawInput []byte,
	only []string,
	llmProfiles []artifacts.LLMProfileManifest,
	llmProfileOverride string,
	sessionID string,
	runtimeOverrides LLMRuntimeOverrides,
	resume bool,
) (pipeline.CheckpointRecorder, pipeline.CheckpointLoader, error) {
	if !settings.Enabled {
		if resume {
			return nil, nil, fmt.Errorf("--resume requires cache.checkpoints.enabled: true")
		}
		return pipeline.NoopCheckpointRecorder(), pipeline.NoopCheckpointLoader(), nil
	}
	identity, err := checkpoint.NewIdentity(checkpoint.IdentityInput{
		Pipeline:         resolved,
		InputKey:         resolved.Input.Module,
		RawInputDigest:   rawInputDigest(rawInput),
		SelectedLanes:    only,
		RuntimeOverrides: runtimeOverrideFingerprints(llmProfileOverride, sessionID, runtimeOverrides),
		References:       pipeline.ReferenceProvenance(resolved),
		ProvenanceFingerprints: combineCheckpointFingerprints(
			llmProfileFingerprints(llmProfiles),
			llmFingerprints,
			checkpointIdentityFingerprints(componentFingerprints),
		),
	})
	if err != nil {
		return nil, nil, fmt.Errorf("create checkpoint identity: %w", err)
	}
	checkpointRoot := strings.TrimSpace(settings.Directory)
	if checkpointRoot == "" {
		checkpointRoot, err = config.DefaultCheckpointRoot(opts.UserCacheDir)
		if err != nil {
			return nil, nil, fmt.Errorf("resolve checkpoint root: %w", err)
		}
	}
	recorder, err := checkpoint.NewFilesystemRecorder(checkpointRoot, identity)
	if err != nil {
		return nil, nil, fmt.Errorf("create checkpoint recorder: %w", err)
	}
	loader := pipeline.NoopCheckpointLoader()
	if resume {
		loader, err = checkpoint.NewFilesystemLoader(checkpointRoot, identity)
		if err != nil {
			return nil, nil, fmt.Errorf("create checkpoint loader: %w", err)
		}
	}
	return recorder, loader, nil
}

func llmCheckpointFingerprints(client contracts.StructuredLLMClient) ([]checkpoint.Fingerprint, error) {
	provider, ok := client.(frameworkllm.CheckpointFingerprintProvider)
	if !ok {
		return nil, nil
	}
	values, err := provider.LLMCheckpointFingerprints()
	if err != nil {
		return nil, err
	}
	out := make([]checkpoint.Fingerprint, 0, len(values))
	for _, value := range values {
		out = append(out, checkpoint.Fingerprint{Name: value.Name, Value: value.Value})
	}
	return out, nil
}

func combineCheckpointFingerprints(sources ...[]checkpoint.Fingerprint) []checkpoint.Fingerprint {
	var out []checkpoint.Fingerprint
	for _, source := range sources {
		out = append(out, source...)
	}
	return out
}

func recomputePolicy(resolved pipeline.ResolvedPipeline, requestedStep string) (pipeline.CheckpointExecutionPolicy, error) {
	requestedStep = strings.TrimSpace(requestedStep)
	if requestedStep == "" {
		return pipeline.CheckpointExecutionPolicy{}, fmt.Errorf("--recompute-step must not be empty")
	}
	var selected *pipeline.ResolvedPipelineStep
	for index := range resolved.Steps {
		if strings.TrimSpace(resolved.Steps[index].ID) == requestedStep {
			selected = &resolved.Steps[index]
			break
		}
	}
	if selected == nil {
		return pipeline.CheckpointExecutionPolicy{}, fmt.Errorf("unknown pipeline step %q", requestedStep)
	}

	policy := pipeline.CheckpointExecutionPolicy{ForcedLanes: make(map[string]struct{}), RequireReusableLanes: make(map[string]struct{})}
	var forced []pipeline.ResolvedArtifactLane
	for _, lane := range selected.ArtifactLanes {
		lane.StepID = selected.ID
		policy.ForcedLanes[pipeline.CheckpointLaneKey(selected.ID, lane.ID)] = struct{}{}
		forced = append(forced, lane)
	}

	lanes := make(map[string]pipeline.ResolvedArtifactLane)
	dependents := make(map[string][]pipeline.ResolvedArtifactLane)
	for _, step := range resolved.Steps {
		for _, lane := range step.ArtifactLanes {
			lane.StepID = step.ID
			lanes[pipeline.CheckpointLaneKey(step.ID, lane.ID)] = lane
			for _, target := range []pipeline.ResolvedReferenceTarget{lane.ExtractReferences, lane.MergeReferences, lane.NormalizeReferences} {
				for _, binding := range target.Bindings {
					if binding.Artifact != nil {
						producerKey := pipeline.CheckpointLaneKey(binding.Artifact.Step, binding.Artifact.Lane)
						dependents[producerKey] = append(dependents[producerKey], lane)
					}
				}
			}
		}
	}
	processed := make(map[string]struct{})
	queue := append([]pipeline.ResolvedArtifactLane(nil), forced...)
	for len(queue) > 0 {
		lane := queue[0]
		queue = queue[1:]
		key := pipeline.CheckpointLaneKey(lane.StepID, lane.ID)
		if _, ok := processed[key]; ok {
			continue
		}
		processed[key] = struct{}{}
		for _, dependent := range dependents[key] {
			dependentKey := pipeline.CheckpointLaneKey(dependent.StepID, dependent.ID)
			if _, alreadyForced := policy.ForcedLanes[dependentKey]; alreadyForced {
				continue
			}
			policy.ForcedLanes[dependentKey] = struct{}{}
			forced = append(forced, dependent)
			queue = append(queue, dependent)
		}
	}

	processed = make(map[string]struct{})
	queue = append([]pipeline.ResolvedArtifactLane(nil), forced...)
	for len(queue) > 0 {
		lane := queue[0]
		queue = queue[1:]
		key := pipeline.CheckpointLaneKey(lane.StepID, lane.ID)
		if _, ok := processed[key]; ok {
			continue
		}
		processed[key] = struct{}{}
		for _, target := range []pipeline.ResolvedReferenceTarget{lane.ExtractReferences, lane.MergeReferences, lane.NormalizeReferences} {
			for _, binding := range target.Bindings {
				if binding.Artifact == nil {
					continue
				}
				producerKey := pipeline.CheckpointLaneKey(binding.Artifact.Step, binding.Artifact.Lane)
				if _, forcedAlready := policy.ForcedLanes[producerKey]; !forcedAlready {
					policy.RequireReusableLanes[producerKey] = struct{}{}
				}
				if producer, ok := lanes[producerKey]; ok {
					queue = append(queue, producer)
				}
			}
		}
	}
	return policy, nil
}

func checkpointIdentityFingerprints(values []pipeline.CheckpointFingerprint) []checkpoint.Fingerprint {
	if len(values) == 0 {
		return nil
	}
	out := make([]checkpoint.Fingerprint, len(values))
	for index, value := range values {
		out[index] = checkpoint.Fingerprint{Name: "component:" + value.Name, Value: value.Value}
	}
	return out
}

func rawInputDigest(data []byte) string {
	sum := sha256.Sum256(data)
	return "sha256:" + hex.EncodeToString(sum[:])
}

func runtimeOverrideFingerprints(llmProfileOverride string, sessionID string, runtimeOverrides LLMRuntimeOverrides) []checkpoint.Fingerprint {
	var values []checkpoint.Fingerprint
	if strings.TrimSpace(llmProfileOverride) != "" {
		values = append(values, checkpoint.Fingerprint{Name: "llm_profile_override", Value: strings.TrimSpace(llmProfileOverride)})
	}
	if strings.TrimSpace(sessionID) != "" {
		values = append(values, checkpoint.Fingerprint{Name: "session_id", Value: strings.TrimSpace(sessionID)})
	}
	if runtimeOverrides.ReasoningEffort != nil {
		value := strings.TrimSpace(*runtimeOverrides.ReasoningEffort)
		if value == "" {
			value = "<cleared>"
		}
		values = append(values, checkpoint.Fingerprint{Name: "reasoning_effort_override", Value: value})
	}
	return values
}

func llmProfileFingerprints(profiles []artifacts.LLMProfileManifest) []checkpoint.Fingerprint {
	if len(profiles) == 0 {
		return nil
	}
	values := make([]checkpoint.Fingerprint, 0, len(profiles))
	for _, profile := range profiles {
		id := strings.TrimSpace(profile.ID)
		if id == "" {
			continue
		}
		values = append(values, checkpoint.Fingerprint{
			Name:  "llm_profile:" + id,
			Value: strings.TrimSpace(profile.Provider) + ":" + strings.TrimSpace(profile.Model),
		})
	}
	return values
}

func configSource(configPath string) string {
	if strings.TrimSpace(configPath) != "" {
		return "flag"
	}
	return "discovered"
}

func writeOutputFiles(runOutputDir string, files []contracts.OutputFile) error {
	for _, file := range files {
		if _, err := outputFilePath(runOutputDir, file.Name); err != nil {
			return err
		}
	}

	outputParent := filepath.Dir(runOutputDir)
	if err := os.MkdirAll(outputParent, 0o755); err != nil {
		return fmt.Errorf("create output parent %q: %w", outputParent, err)
	}
	if err := os.Mkdir(runOutputDir, 0o755); err != nil {
		if os.IsExist(err) {
			return fmt.Errorf("output run directory %q already exists", runOutputDir)
		}
		return fmt.Errorf("create output run directory %q: %w", runOutputDir, err)
	}
	for _, file := range files {
		if err := fileio.WriteBytes(runOutputDir, file.Name, file.Bytes, 0o755, 0o644); err != nil {
			return fmt.Errorf("write output file %q: %w", file.Name, err)
		}
	}
	return nil
}

func outputFilePath(runOutputDir, logicalName string) (string, error) {
	name := strings.TrimSpace(logicalName)
	if name == "" {
		return "", fmt.Errorf("output file name must not be empty")
	}
	if strings.Contains(name, `\`) {
		return "", fmt.Errorf("output file name %q must use slash-separated relative paths", name)
	}
	if path.IsAbs(name) || filepath.IsAbs(name) {
		return "", fmt.Errorf("output file name %q must be relative", name)
	}
	if strings.Contains(name, "..") {
		return "", fmt.Errorf("output file name %q must not contain ..", name)
	}
	cleaned := path.Clean(name)
	if cleaned == "." || cleaned != name {
		return "", fmt.Errorf("output file name %q must be clean", name)
	}

	root, err := filepath.Abs(runOutputDir)
	if err != nil {
		return "", fmt.Errorf("resolve output directory %q: %w", runOutputDir, err)
	}
	target, err := filepath.Abs(filepath.Join(root, filepath.FromSlash(cleaned)))
	if err != nil {
		return "", fmt.Errorf("resolve output file %q: %w", name, err)
	}
	rel, err := filepath.Rel(root, target)
	if err != nil {
		return "", fmt.Errorf("resolve output file %q: %w", name, err)
	}
	if rel == "." || strings.HasPrefix(rel, ".."+string(filepath.Separator)) || rel == ".." {
		return "", fmt.Errorf("output file name %q resolves outside output directory", name)
	}
	return target, nil
}

func reorderRunArgs(args []string) []string {
	var flags []string
	var positionals []string
	for i := 0; i < len(args); i++ {
		arg := args[i]
		if arg == "--" {
			positionals = append(positionals, args[i+1:]...)
			break
		}
		if strings.HasPrefix(arg, "-") {
			flags = append(flags, arg)
			if runFlagTakesValue(arg) && !strings.Contains(arg, "=") && i+1 < len(args) {
				i++
				flags = append(flags, args[i])
			}
			continue
		}
		positionals = append(positionals, arg)
	}
	return append(flags, positionals...)
}

func runFlagTakesValue(arg string) bool {
	switch arg {
	case "--config", "--input", "--only", "--output-dir", "--debug-dir", "--llm-profile", "--session-id", "--reasoning-effort", "--chunk_cache", "--reference", "--without-reference", "--recompute-step":
		return true
	default:
		return false
	}
}

type chunkCacheFlag struct {
	value pipeline.ChunkCacheMode
	set   bool
}

func (f *chunkCacheFlag) String() string {
	if f == nil {
		return ""
	}
	return string(f.value)
}

func (f *chunkCacheFlag) Set(raw string) error {
	mode, err := pipeline.ParseChunkCacheMode(raw)
	if err != nil {
		return err
	}
	f.value = mode
	f.set = true
	return nil
}

func (f chunkCacheFlag) explicitValue() string {
	if !f.set {
		return ""
	}
	return string(f.value)
}

func chunkPlanStoreForRun(cfg config.ChunkPlanCacheConfig, opts Options) (pipeline.ChunkPlanStore, error) {
	if cfg.Mode == pipeline.ChunkCacheBypass {
		return nil, nil
	}
	root := strings.TrimSpace(cfg.Directory)
	if root == "" {
		var err error
		root, err = config.DefaultChunkPlanRoot(opts.UserCacheDir)
		if err != nil {
			return nil, fmt.Errorf("resolve chunk plan root: %w", err)
		}
	}
	store, err := opts.ChunkPlanStoreFactory(root)
	if err != nil {
		return nil, fmt.Errorf("create chunk plan store at %q: %w", root, err)
	}
	if store == nil {
		return nil, fmt.Errorf("create chunk plan store at %q: factory returned nil", root)
	}
	return store, nil
}

func validateRunFlagValues(args []string) error {
	for i, arg := range args {
		if arg != "--session-id" && arg != "--reasoning-effort" {
			continue
		}
		if i+1 >= len(args) || strings.HasPrefix(args[i+1], "-") {
			return fmt.Errorf("flag needs an argument: %s", arg)
		}
	}
	return nil
}

func flagWasProvided(args []string, name string) bool {
	for _, arg := range args {
		if arg == name || strings.HasPrefix(arg, name+"=") {
			return true
		}
	}
	return false
}

func effectiveLLMProfileIDs(resolved pipeline.ResolvedPipeline) []string {
	seen := make(map[string]struct{})
	add := func(binding pipeline.ModuleBinding) {
		id := strings.TrimSpace(binding.LLMProfile)
		if id != "" {
			seen[id] = struct{}{}
		}
	}
	if resolved.InputExecutionClass == contracts.ExecutionClassLLMBacked {
		add(resolved.Input)
	}
	if resolved.ChunkExecutionClass == contracts.ExecutionClassLLMBacked {
		add(resolved.Chunk)
	}
	for _, lane := range resolved.AllArtifactLanes() {
		if lane.ExtractExecutionClass == contracts.ExecutionClassLLMBacked {
			add(lane.Extract)
		}
		if lane.MergeExecutionClass == contracts.ExecutionClassLLMBacked {
			add(lane.Merge)
		}
		if lane.NormalizeExecutionClass == contracts.ExecutionClassLLMBacked {
			add(lane.Normalize)
		}
	}
	for _, chain := range resolved.ValidatorChains {
		for _, validator := range chain.Validators {
			if validator.ExecutionClass == contracts.ExecutionClassLLMBacked {
				add(validator.Binding)
			}
		}
	}
	if resolved.OutputExecutionClass == contracts.ExecutionClassLLMBacked {
		add(resolved.Output)
	}
	ids := make([]string, 0, len(seen))
	for id := range seen {
		ids = append(ids, id)
	}
	sort.Strings(ids)
	return ids
}

func runMetadata(outputDir, debugDir string) map[string]any {
	metadata := make(map[string]any)
	if dir := strings.TrimSpace(outputDir); dir != "" {
		metadata["output_dir"] = dir
	}
	if dir := strings.TrimSpace(debugDir); dir != "" {
		metadata["debug_dir"] = dir
	}
	if len(metadata) == 0 {
		return nil
	}
	return metadata
}

func runConfig(args []string, stdout, stderr io.Writer, opts Options) int {
	if len(args) == 0 {
		fmt.Fprintln(stderr, "notarius: config requires a subcommand")
		writeUsage(stderr)
		return 2
	}
	switch args[0] {
	case "validate":
		return runConfigValidate(args[1:], stdout, stderr, opts)
	default:
		fmt.Fprintf(stderr, "notarius: unknown config subcommand %q\n", args[0])
		writeUsage(stderr)
		return 2
	}
}

func runConfigValidate(args []string, stdout, stderr io.Writer, opts Options) int {
	fs := flag.NewFlagSet("config validate", flag.ContinueOnError)
	fs.SetOutput(io.Discard)
	configPath := fs.String("config", "", "config file path")
	pipelineID := fs.String("pipeline", "", "pipeline ID")
	onlyRaw := fs.String("only", "", "comma-separated artifact lanes")
	if err := fs.Parse(args); err != nil {
		fmt.Fprintf(stderr, "notarius: %v\n", err)
		return 2
	}
	if fs.NArg() != 0 {
		fmt.Fprintf(stderr, "notarius: unexpected argument %q\n", fs.Arg(0))
		return 2
	}
	if strings.TrimSpace(*onlyRaw) != "" && strings.TrimSpace(*pipelineID) == "" {
		fmt.Fprintln(stderr, "notarius: --only requires --pipeline")
		return 2
	}
	only, err := parseOnly(*onlyRaw)
	if err != nil {
		fmt.Fprintf(stderr, "notarius: %v\n", err)
		return 2
	}

	cfg, path, err := loadConfig(*configPath, opts)
	if err != nil {
		fmt.Fprintf(stderr, "notarius: %v\n", err)
		return 1
	}

	if strings.TrimSpace(*pipelineID) != "" {
		catalog, err := effectiveCatalog(opts)
		if err != nil {
			fmt.Fprintf(stderr, "notarius: %v\n", err)
			return 1
		}
		effective, err := cfg.Resolve(config.ResolveInput{
			PipelineID: *pipelineID,
			Only:       only,
			Catalog:    catalog,
		})
		if err != nil {
			fmt.Fprintf(stderr, "notarius: %v\n", err)
			return 1
		}
		if err := validateExplicitPromptKitProfiles(context.Background(), effective.Config, effectiveLLMProfileIDs(effective.ResolvedPipeline), opts.promptKitAssets); err != nil {
			fmt.Fprintf(stderr, "notarius: %v\n", err)
			return 1
		}
		fmt.Fprintf(stdout, "config %q is valid for pipeline %q\n", path, strings.TrimSpace(*pipelineID))
		return 0
	}

	if err := cfg.Validate(); err != nil {
		fmt.Fprintf(stderr, "notarius: %v\n", err)
		return 1
	}
	fmt.Fprintf(stdout, "config %q is valid\n", path)
	return 0
}

func runPipelines(args []string, stdout, stderr io.Writer, opts Options) int {
	if len(args) == 0 {
		fmt.Fprintln(stderr, "notarius: pipelines requires a subcommand")
		writeUsage(stderr)
		return 2
	}
	switch args[0] {
	case "list":
		return runPipelinesList(args[1:], stdout, stderr, opts)
	default:
		fmt.Fprintf(stderr, "notarius: unknown pipelines subcommand %q\n", args[0])
		writeUsage(stderr)
		return 2
	}
}

func runPipelinesList(args []string, stdout, stderr io.Writer, opts Options) int {
	fs := flag.NewFlagSet("pipelines list", flag.ContinueOnError)
	fs.SetOutput(io.Discard)
	configPath := fs.String("config", "", "config file path")
	jsonOutput := fs.Bool("json", false, "write JSON output")
	if err := fs.Parse(args); err != nil {
		fmt.Fprintf(stderr, "notarius: %v\n", err)
		return 2
	}
	if fs.NArg() != 0 {
		fmt.Fprintf(stderr, "notarius: unexpected argument %q\n", fs.Arg(0))
		return 2
	}

	cfg, _, err := loadConfig(*configPath, opts)
	if err != nil {
		fmt.Fprintf(stderr, "notarius: %v\n", err)
		return 1
	}
	if err := cfg.Validate(); err != nil {
		fmt.Fprintf(stderr, "notarius: %v\n", err)
		return 1
	}

	ids := sortedPipelineIDs(cfg)
	if *jsonOutput {
		payload := struct {
			Pipelines []string `json:"pipelines"`
		}{Pipelines: ids}
		encoded, err := json.Marshal(payload)
		if err != nil {
			fmt.Fprintf(stderr, "notarius: marshal pipeline list: %v\n", err)
			return 1
		}
		fmt.Fprintf(stdout, "%s\n", encoded)
		return 0
	}

	for _, id := range ids {
		fmt.Fprintln(stdout, id)
	}
	return 0
}

func loadConfig(configPath string, opts Options) (config.Config, string, error) {
	path, err := discoverConfigPath(configPath, opts)
	if err != nil {
		return config.Config{}, "", err
	}

	fileCfg, err := config.LoadFileConfig(path)
	if err != nil {
		return config.Config{}, "", err
	}
	cfg := config.Default()
	if err := cfg.ApplyFileConfigWithLookup(fileCfg, opts.LookupEnv); err != nil {
		return config.Config{}, "", err
	}
	if err := cfg.ApplyEnvOverridesWithLookup(opts.LookupEnv); err != nil {
		return config.Config{}, "", err
	}
	return cfg, path, nil
}

func discoverConfigPath(configPath string, opts Options) (string, error) {
	if path := strings.TrimSpace(configPath); path != "" {
		if err := requireConfigFile(path); err != nil {
			return "", err
		}
		return path, nil
	}
	if path, ok := opts.LookupEnv("NOTARIUS_CONFIG"); ok && strings.TrimSpace(path) != "" {
		path = strings.TrimSpace(path)
		if err := requireConfigFile(path); err != nil {
			return "", err
		}
		return path, nil
	}
	if _, err := os.Stat(defaultConfigPath); err == nil {
		return defaultConfigPath, nil
	} else if err != nil && !os.IsNotExist(err) {
		return "", fmt.Errorf("check default config %q: %w", defaultConfigPath, err)
	}
	return "", fmt.Errorf("config file not found; pass --config or set NOTARIUS_CONFIG")
}

func requireConfigFile(path string) error {
	info, err := os.Stat(path)
	if err != nil {
		return fmt.Errorf("config file %q is not available: %w", path, err)
	}
	if info.IsDir() {
		return fmt.Errorf("config file %q is a directory", path)
	}
	return nil
}

func parseOnly(raw string) ([]string, error) {
	if strings.TrimSpace(raw) == "" {
		return nil, nil
	}
	parts := strings.Split(raw, ",")
	result := make([]string, 0, len(parts))
	for _, part := range parts {
		trimmed := strings.TrimSpace(part)
		if trimmed == "" {
			return nil, fmt.Errorf("--only must contain comma-separated non-empty artifact lane IDs")
		}
		result = append(result, trimmed)
	}
	return result, nil
}

type stringListFlag []string

func (flag *stringListFlag) String() string {
	if flag == nil {
		return ""
	}
	return strings.Join(*flag, ",")
}

func (flag *stringListFlag) Set(value string) error {
	*flag = append(*flag, value)
	return nil
}

type sessionIDFlag struct {
	value string
	set   bool
}

type singleValueFlag struct {
	name  string
	value string
	set   bool
}

func (flag *singleValueFlag) String() string {
	if flag == nil {
		return ""
	}
	return flag.value
}

func (flag *singleValueFlag) Set(value string) error {
	if flag.set {
		return fmt.Errorf("%s may be specified only once", flag.name)
	}
	flag.value = value
	flag.set = true
	return nil
}

func (flag *sessionIDFlag) String() string {
	if flag == nil {
		return ""
	}
	return flag.value
}

func (flag *sessionIDFlag) Set(value string) error {
	flag.value = value
	flag.set = true
	return nil
}

type cliReferenceRequest struct {
	Selector cliReferenceSelector
	Source   string
}

type cliReferenceUnbindRequest struct {
	Selector cliReferenceSelector
}

type cliReferenceSelector struct {
	LaneID   string
	Stage    pipeline.ModuleStage
	SlotName string
}

func parseReferenceFlags(values []string) ([]cliReferenceRequest, error) {
	if len(values) == 0 {
		return nil, nil
	}
	requests := make([]cliReferenceRequest, 0, len(values))
	for _, raw := range values {
		name, source, ok := strings.Cut(raw, "=")
		if !ok {
			return nil, fmt.Errorf("--reference must use slot=path or lane.slot=path")
		}
		if strings.TrimSpace(source) == "" {
			return nil, fmt.Errorf("--reference path must not be empty; use --without-reference to unbind")
		}
		selector, err := parseReferenceSelector(name, "--reference")
		if err != nil {
			return nil, err
		}
		requests = append(requests, cliReferenceRequest{
			Selector: selector,
			Source:   strings.TrimSpace(source),
		})
	}
	return requests, nil
}

func parseReferenceUnbindFlags(values []string) ([]cliReferenceUnbindRequest, error) {
	if len(values) == 0 {
		return nil, nil
	}
	requests := make([]cliReferenceUnbindRequest, 0, len(values))
	for _, raw := range values {
		if strings.Contains(raw, "=") {
			return nil, fmt.Errorf("--without-reference must use a reference selector without =path")
		}
		selector, err := parseReferenceSelector(raw, "--without-reference")
		if err != nil {
			return nil, err
		}
		requests = append(requests, cliReferenceUnbindRequest{
			Selector: selector,
		})
	}
	return requests, nil
}

func parseReferenceSelector(raw string, flagName string) (cliReferenceSelector, error) {
	selector := strings.TrimSpace(raw)
	if selector == "" {
		return cliReferenceSelector{}, fmt.Errorf("%s reference slot must not be empty", flagName)
	}
	parts := strings.Split(selector, ".")
	for _, part := range parts {
		if strings.TrimSpace(part) == "" {
			return cliReferenceSelector{}, fmt.Errorf("%s must use non-empty reference selector values", flagName)
		}
	}
	switch len(parts) {
	case 1:
		return cliReferenceSelector{SlotName: strings.TrimSpace(parts[0])}, nil
	case 2:
		first := strings.TrimSpace(parts[0])
		slotName := strings.TrimSpace(parts[1])
		if first == string(pipeline.StageChunk) {
			return cliReferenceSelector{Stage: pipeline.StageChunk, SlotName: slotName}, nil
		}
		if first == string(pipeline.StageMerge) {
			return cliReferenceSelector{Stage: pipeline.StageMerge, SlotName: slotName}, nil
		}
		return cliReferenceSelector{LaneID: first, SlotName: slotName}, nil
	case 3:
		laneID := strings.TrimSpace(parts[0])
		stage := pipeline.ModuleStage(strings.TrimSpace(parts[1]))
		slotName := strings.TrimSpace(parts[2])
		if stage != pipeline.StageExtract && stage != pipeline.StageMerge && stage != pipeline.StageNormalize {
			return cliReferenceSelector{}, fmt.Errorf("%s lane-qualified selector must use lane.extract.slot, lane.merge.slot, or lane.normalize.slot", flagName)
		}
		return cliReferenceSelector{LaneID: laneID, Stage: stage, SlotName: slotName}, nil
	default:
		return cliReferenceSelector{}, fmt.Errorf("%s must use slot, chunk.slot, merge.slot, lane.slot, lane.extract.slot, lane.merge.slot, or lane.normalize.slot", flagName)
	}
}

func resolveCLIReferenceRequests(
	cfg config.Config,
	pipelineID string,
	only []string,
	catalog pipeline.ModuleCatalog,
	referenceRequests []cliReferenceRequest,
	unbindRequests []cliReferenceUnbindRequest,
) ([]pipeline.ReferenceBinding, []pipeline.ReferenceUnbind, error) {
	if len(referenceRequests) == 0 && len(unbindRequests) == 0 {
		return nil, nil, nil
	}

	targets, err := selectedReferenceTargets(cfg, pipelineID, only, catalog)
	if err != nil {
		return nil, nil, err
	}

	overrides := make([]pipeline.ReferenceBinding, 0, len(referenceRequests))
	for _, request := range referenceRequests {
		target, err := resolveCLIReferenceTarget(targets, request.Selector)
		if err != nil {
			return nil, nil, err
		}
		overrides = append(overrides, pipeline.ReferenceBinding{
			Stage:         target.stage,
			LaneID:        target.laneID,
			SlotName:      request.Selector.SlotName,
			Source:        request.Source,
			BindingSource: contracts.ReferenceBindingSourceCLI,
		})
	}

	unbinds := make([]pipeline.ReferenceUnbind, 0, len(unbindRequests))
	for _, request := range unbindRequests {
		target, err := resolveCLIReferenceTarget(targets, request.Selector)
		if err != nil {
			return nil, nil, err
		}
		unbinds = append(unbinds, pipeline.ReferenceUnbind{
			Stage:    target.stage,
			LaneID:   target.laneID,
			SlotName: request.Selector.SlotName,
		})
	}

	return overrides, unbinds, nil
}

type selectedReferenceTarget struct {
	laneID string
	stage  pipeline.ModuleStage
	module string
	slots  map[string]struct{}
}

func selectedReferenceTargets(cfg config.Config, pipelineID string, only []string, catalog pipeline.ModuleCatalog) ([]selectedReferenceTarget, error) {
	profile, ok := lookupCLIReferencePipeline(cfg.Pipelines, pipelineID)
	if !ok {
		return nil, fmt.Errorf("pipeline %q is not configured", strings.TrimSpace(pipelineID))
	}
	if profile.Steps != nil && len(only) > 0 {
		return nil, fmt.Errorf("pipeline %q --only is not supported for explicit ordered steps", strings.TrimSpace(pipelineID))
	}

	lanesByID := make(map[string]pipeline.ArtifactLaneProfile, len(profile.Artifacts))
	addLanes := func(values map[string]pipeline.ArtifactLaneProfile) error {
		for rawLaneID, lane := range values {
			laneID := strings.TrimSpace(rawLaneID)
			if laneID == "" {
				return fmt.Errorf("pipeline %q artifact lane id must not be empty", strings.TrimSpace(pipelineID))
			}
			if _, ok := lanesByID[laneID]; ok {
				return fmt.Errorf("pipeline %q artifact lane %q is duplicated after trimming", strings.TrimSpace(pipelineID), laneID)
			}
			lanesByID[laneID] = lane
		}
		return nil
	}
	if profile.Steps == nil {
		if err := addLanes(profile.Artifacts); err != nil {
			return nil, err
		}
	} else {
		for _, step := range profile.Steps {
			if err := addLanes(step.Artifacts); err != nil {
				return nil, err
			}
		}
	}

	selectedIDs := make([]string, 0, len(lanesByID))
	if len(only) == 0 {
		for laneID := range lanesByID {
			selectedIDs = append(selectedIDs, laneID)
		}
	} else {
		seen := make(map[string]struct{}, len(only))
		for _, rawLaneID := range only {
			laneID := strings.TrimSpace(rawLaneID)
			if laneID == "" {
				return nil, fmt.Errorf("pipeline %q selected artifact lane id must not be empty", strings.TrimSpace(pipelineID))
			}
			if _, ok := lanesByID[laneID]; !ok {
				return nil, fmt.Errorf("pipeline %q selected artifact lane %q is not declared", strings.TrimSpace(pipelineID), laneID)
			}
			if _, ok := seen[laneID]; !ok {
				selectedIDs = append(selectedIDs, laneID)
				seen[laneID] = struct{}{}
			}
		}
	}
	sort.Strings(selectedIDs)

	targets := make([]selectedReferenceTarget, 0, 1+len(selectedIDs)*3)
	chunk := pipeline.Binding(profile.Chunk.Module)
	chunk.Module = strings.TrimSpace(profile.Chunk.Module)
	if chunk.Module == "" {
		chunk.Module = pipeline.DefaultChunkModule
	}
	chunkSpec, err := cliReferenceChunkerSpec(catalog, chunk.Module)
	if err != nil {
		return nil, fmt.Errorf("pipeline %q chunk module %q: %w", strings.TrimSpace(pipelineID), chunk.Module, err)
	}
	targets = append(targets, selectedReferenceTarget{
		stage:  pipeline.StageChunk,
		module: chunk.Module,
		slots:  referenceSlotSet(chunkSpec.ReferenceSlots),
	})

	for _, laneID := range selectedIDs {
		lane := lanesByID[laneID]
		extractModule := strings.TrimSpace(lane.Extract.Module)
		if extractModule == "" {
			return nil, fmt.Errorf("pipeline %q lane %q extract module must not be empty", strings.TrimSpace(pipelineID), laneID)
		}
		extractSpec, err := cliReferenceExtractorSpec(catalog, extractModule)
		if err != nil {
			return nil, fmt.Errorf("pipeline %q lane %q extract module %q: %w", strings.TrimSpace(pipelineID), laneID, extractModule, err)
		}
		artifactKind := extractSpec.ArtifactKind
		if artifactKind == "" {
			return nil, fmt.Errorf("pipeline %q lane %q extract module %q does not declare an artifact kind", strings.TrimSpace(pipelineID), laneID, extractModule)
		}
		targets = append(targets, selectedReferenceTarget{
			laneID: laneID,
			stage:  pipeline.StageExtract,
			module: extractModule,
			slots:  referenceSlotSet(extractSpec.ReferenceSlots),
		})

		mergeModule := strings.TrimSpace(lane.Merge.Module)
		if mergeModule == "" {
			mergeModule = pipeline.DefaultMergeModule
		}
		mergeSpec, err := cliReferenceMergerSpec(catalog, mergeModule, artifactKind)
		if err != nil {
			return nil, fmt.Errorf("pipeline %q lane %q merge module %q: %w", strings.TrimSpace(pipelineID), laneID, mergeModule, err)
		}
		targets = append(targets, selectedReferenceTarget{
			laneID: laneID,
			stage:  pipeline.StageMerge,
			module: mergeModule,
			slots:  referenceSlotSet(mergeSpec.ReferenceSlots),
		})

		normalizeModule := strings.TrimSpace(lane.Normalize.Module)
		if normalizeModule == "" {
			normalizeModule = pipeline.DefaultNormalizeModule
		}
		normalizeSpec, err := cliReferenceNormalizerSpec(catalog, normalizeModule, artifactKind)
		if err != nil {
			return nil, fmt.Errorf("pipeline %q lane %q normalize module %q: %w", strings.TrimSpace(pipelineID), laneID, normalizeModule, err)
		}
		targets = append(targets, selectedReferenceTarget{
			laneID: laneID,
			stage:  pipeline.StageNormalize,
			module: normalizeModule,
			slots:  referenceSlotSet(normalizeSpec.ReferenceSlots),
		})
	}
	return targets, nil
}

func lookupCLIReferencePipeline(profiles map[string]pipeline.PipelineProfile, pipelineID string) (pipeline.PipelineProfile, bool) {
	pipelineID = strings.TrimSpace(pipelineID)
	for rawID, profile := range profiles {
		if strings.TrimSpace(rawID) == pipelineID {
			return profile, true
		}
	}
	return pipeline.PipelineProfile{}, false
}

func cliReferenceChunkerSpec(catalog pipeline.ModuleCatalog, module string) (pipeline.ModuleSpec, error) {
	if catalog.Chunkers == nil {
		return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
	}
	spec, ok := catalog.Chunkers.Spec(module)
	if !ok {
		return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
	}
	return spec, nil
}

func cliReferenceExtractorSpec(catalog pipeline.ModuleCatalog, module string) (pipeline.ModuleSpec, error) {
	if catalog.Extractors == nil {
		return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
	}
	spec, ok := catalog.Extractors.Spec(module)
	if !ok {
		return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
	}
	return spec, nil
}

func cliReferenceMergerSpec(catalog pipeline.ModuleCatalog, module string, kind contracts.ArtifactKind) (pipeline.ModuleSpec, error) {
	if catalog.Mergers == nil {
		return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
	}
	registered := catalog.Mergers.RegisteredArtifactKinds(module)
	if len(registered) == 0 {
		return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
	}
	spec, ok := catalog.Mergers.SpecForArtifact(module, kind)
	if !ok {
		return pipeline.ModuleSpec{}, cliReferenceArtifactVariantError("merger", module, kind, registered)
	}
	return spec, nil
}

func cliReferenceNormalizerSpec(catalog pipeline.ModuleCatalog, module string, kind contracts.ArtifactKind) (pipeline.ModuleSpec, error) {
	if catalog.Normalizers == nil {
		return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
	}
	registered := catalog.Normalizers.RegisteredArtifactKinds(module)
	if len(registered) == 0 {
		return pipeline.ModuleSpec{}, fmt.Errorf("module %q is not registered", module)
	}
	spec, ok := catalog.Normalizers.SpecForArtifact(module, kind)
	if !ok {
		return pipeline.ModuleSpec{}, cliReferenceArtifactVariantError("normalizer", module, kind, registered)
	}
	return spec, nil
}

func cliReferenceArtifactVariantError(moduleType string, module string, kind contracts.ArtifactKind, registered []contracts.ArtifactKind) error {
	values := make([]string, len(registered))
	for i, value := range registered {
		values[i] = string(value)
	}
	if len(values) == 0 {
		return fmt.Errorf("%s %q has no typed variant for artifact kind %q", moduleType, module, kind)
	}
	return fmt.Errorf("%s %q has no typed variant for artifact kind %q; registered kinds: %s", moduleType, module, kind, strings.Join(values, ", "))
}

func referenceSlotSet(slots []contracts.ReferenceSlot) map[string]struct{} {
	slotSet := make(map[string]struct{}, len(slots))
	for _, slot := range slots {
		slotSet[slot.Name] = struct{}{}
	}
	return slotSet
}

func resolveCLIReferenceTarget(targets []selectedReferenceTarget, selector cliReferenceSelector) (selectedReferenceTarget, error) {
	slotName := strings.TrimSpace(selector.SlotName)
	if slotName == "" {
		return selectedReferenceTarget{}, fmt.Errorf("reference slot must not be empty")
	}
	if selector.Stage == pipeline.StageChunk {
		for _, target := range targets {
			if target.stage != pipeline.StageChunk {
				continue
			}
			if _, ok := target.slots[slotName]; !ok {
				return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by chunk module %q", slotName, target.module)
			}
			return target, nil
		}
		return selectedReferenceTarget{}, fmt.Errorf("reference chunk target is not selected")
	}
	if selector.Stage == pipeline.StageExtract || selector.Stage == pipeline.StageMerge || selector.Stage == pipeline.StageNormalize {
		if selector.LaneID == "" && selector.Stage == pipeline.StageMerge {
			return resolveCLIReferenceStageTarget(targets, selector.Stage, slotName)
		}
		for _, target := range targets {
			if target.laneID == selector.LaneID && target.stage == selector.Stage {
				if _, ok := target.slots[slotName]; !ok {
					return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by selected %s target %q", slotName, selector.Stage, targetLabel(target))
				}
				return target, nil
			}
		}
		return selectedReferenceTarget{}, fmt.Errorf("reference lane %q is not selected", selector.LaneID)
	}
	if strings.TrimSpace(selector.LaneID) != "" {
		return resolveCLIReferenceLaneTarget(targets, strings.TrimSpace(selector.LaneID), slotName)
	}
	return resolveCLIReferenceFlatTarget(targets, slotName)
}

func resolveCLIReferenceStageTarget(targets []selectedReferenceTarget, stage pipeline.ModuleStage, slotName string) (selectedReferenceTarget, error) {
	matches := make([]selectedReferenceTarget, 0, 2)
	for _, target := range targets {
		if target.stage != stage {
			continue
		}
		if _, ok := target.slots[slotName]; ok {
			matches = append(matches, target)
		}
	}
	switch len(matches) {
	case 0:
		return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by any selected %s target", slotName, stage)
	case 1:
		return matches[0], nil
	default:
		return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is declared by multiple selected %s targets (%s); use a more specific selector such as %s", slotName, stage, targetList(matches), selectorSuggestions(matches, slotName))
	}
}

func resolveCLIReferenceLaneTarget(targets []selectedReferenceTarget, laneID string, slotName string) (selectedReferenceTarget, error) {
	laneSelected := false
	matches := make([]selectedReferenceTarget, 0, 2)
	for _, target := range targets {
		if target.laneID != laneID {
			continue
		}
		laneSelected = true
		if _, ok := target.slots[slotName]; ok {
			matches = append(matches, target)
		}
	}
	if !laneSelected {
		return selectedReferenceTarget{}, fmt.Errorf("reference lane %q is not selected", laneID)
	}
	switch len(matches) {
	case 0:
		return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by selected lane %q", slotName, laneID)
	case 1:
		return matches[0], nil
	default:
		return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is declared by multiple selected targets in lane %q (%s); use a more specific selector such as %s", slotName, laneID, targetList(matches), selectorSuggestions(matches, slotName))
	}
}

func resolveCLIReferenceFlatTarget(targets []selectedReferenceTarget, slotName string) (selectedReferenceTarget, error) {
	matches := make([]selectedReferenceTarget, 0, 2)
	for _, target := range targets {
		if _, ok := target.slots[slotName]; ok {
			matches = append(matches, target)
		}
	}
	switch len(matches) {
	case 0:
		return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is not declared by any selected reference target", slotName)
	case 1:
		return matches[0], nil
	default:
		return selectedReferenceTarget{}, fmt.Errorf("reference slot %q is declared by multiple selected targets (%s); use a more specific selector such as %s", slotName, targetList(matches), selectorSuggestions(matches, slotName))
	}
}

func targetList(targets []selectedReferenceTarget) string {
	labels := make([]string, 0, len(targets))
	for _, target := range targets {
		labels = append(labels, targetLabel(target))
	}
	sort.Strings(labels)
	return strings.Join(labels, ", ")
}

func targetLabel(target selectedReferenceTarget) string {
	if target.stage == pipeline.StageChunk {
		return "chunk"
	}
	return target.laneID + "." + string(target.stage)
}

func selectorSuggestions(targets []selectedReferenceTarget, slotName string) string {
	suggestions := make([]string, 0, len(targets))
	for _, target := range targets {
		if target.stage == pipeline.StageChunk {
			suggestions = append(suggestions, "chunk."+slotName)
			continue
		}
		suggestions = append(suggestions, target.laneID+"."+string(target.stage)+"."+slotName)
	}
	sort.Strings(suggestions)
	return strings.Join(suggestions, " or ")
}

func sortedPipelineIDs(cfg config.Config) []string {
	ids := make([]string, 0, len(cfg.Pipelines))
	for id := range cfg.Pipelines {
		ids = append(ids, strings.TrimSpace(id))
	}
	sort.Strings(ids)
	return ids
}