notarius/internal/core/config/file_config.go

package config

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
	"gopkg.in/yaml.v3"
)

type FileConfig struct {
	Version     int                            `yaml:"version"`
	PromptKit   *FilePromptKitConfig           `yaml:"promptkit,omitempty"`
	Pipelines   map[string]FilePipelineProfile `yaml:"pipelines,omitempty"`
	Concurrency *FileConcurrencyConfig         `yaml:"concurrency,omitempty"`
	Output      *FileOutputConfig              `yaml:"output,omitempty"`
	Cache       *FileCacheConfig               `yaml:"cache,omitempty"`
	Debug       *FileDebugConfig               `yaml:"debug,omitempty"`
}

type FilePromptKitConfig struct {
	ProfileDir   *string                          `yaml:"profile_dir,omitempty"`
	ProfileFile  *string                          `yaml:"profile_file,omitempty"`
	LocalBackend *FilePromptKitLocalBackendConfig `yaml:"local_backend,omitempty"`
}

type FilePromptKitLocalBackendConfig struct {
	Endpoint         *string `yaml:"endpoint,omitempty"`
	ConcurrencyLimit *int    `yaml:"concurrency_limit,omitempty"`
}

type FilePipelineProfile struct {
	LLMProfile    *string                            `yaml:"llm_profile,omitempty"`
	Input         fileModuleBinding                  `yaml:"input"`
	Chunk         *fileModuleBinding                 `yaml:"chunk,omitempty"`
	Artifacts     map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
	Steps         []FilePipelineStepProfile          `yaml:"steps,omitempty"`
	Output        *fileModuleBinding                 `yaml:"output,omitempty"`
	References    map[string]fileReferenceSource     `yaml:"references,omitempty"`
	artifactsSet  bool                               `yaml:"-"`
	stepsSet      bool                               `yaml:"-"`
	llmProfileSet bool                               `yaml:"-"`
}

func (p *FilePipelineProfile) UnmarshalYAML(node *yaml.Node) error {
	type plainFilePipelineProfile FilePipelineProfile
	var decoded plainFilePipelineProfile
	seen, err := decodeKnownMapping(node, &decoded, map[string]struct{}{
		"llm_profile": {}, "input": {}, "chunk": {}, "artifacts": {}, "steps": {}, "output": {}, "references": {},
	}, "pipeline profile")
	if err != nil {
		return err
	}
	*p = FilePipelineProfile(decoded)
	_, p.artifactsSet = seen["artifacts"]
	_, p.stepsSet = seen["steps"]
	_, p.llmProfileSet = seen["llm_profile"]
	return nil
}

func (s *FilePipelineStepProfile) UnmarshalYAML(node *yaml.Node) error {
	type plainFilePipelineStepProfile FilePipelineStepProfile
	var decoded plainFilePipelineStepProfile
	if _, err := decodeKnownMapping(node, &decoded, map[string]struct{}{
		"id": {}, "artifacts": {}, "references": {},
	}, "pipeline step"); err != nil {
		return err
	}
	*s = FilePipelineStepProfile(decoded)
	return nil
}

func (l *FileArtifactLaneProfile) UnmarshalYAML(node *yaml.Node) error {
	type plainFileArtifactLaneProfile FileArtifactLaneProfile
	var decoded plainFileArtifactLaneProfile
	if _, err := decodeKnownMapping(node, &decoded, map[string]struct{}{
		"extract": {}, "merge": {}, "normalize": {}, "validators": {}, "references": {},
	}, "artifact lane"); err != nil {
		return err
	}
	*l = FileArtifactLaneProfile(decoded)
	return nil
}

func decodeKnownMapping(node *yaml.Node, target any, allowed map[string]struct{}, context string) (map[string]struct{}, error) {
	if node.Kind != yaml.MappingNode {
		return nil, fmt.Errorf("%s must be an object", context)
	}
	if err := node.Decode(target); err != nil {
		return nil, err
	}
	seen := make(map[string]struct{}, len(node.Content)/2)
	for i := 0; i < len(node.Content); i += 2 {
		key := node.Content[i].Value
		if _, exists := seen[key]; exists {
			return nil, fmt.Errorf("%s field %q is duplicated", context, key)
		}
		if _, ok := allowed[key]; !ok {
			return nil, fmt.Errorf("field %s not found in %s", key, context)
		}
		seen[key] = struct{}{}
	}
	return seen, nil
}

type FilePipelineStepProfile struct {
	ID         string                             `yaml:"id"`
	Artifacts  map[string]FileArtifactLaneProfile `yaml:"artifacts"`
	References map[string]fileReferenceSource     `yaml:"references,omitempty"`
}

type FileArtifactLaneProfile struct {
	Extract    fileModuleBinding              `yaml:"extract"`
	Merge      *fileModuleBinding             `yaml:"merge,omitempty"`
	Normalize  *fileModuleBinding             `yaml:"normalize,omitempty"`
	Validators []fileModuleBinding            `yaml:"validators,omitempty"`
	References map[string]fileReferenceSource `yaml:"references,omitempty"`
}

type FileConcurrencyConfig struct {
	TotalLLM     *int           `yaml:"total_llm,omitempty"`
	StageWorkers map[string]int `yaml:"stage_workers,omitempty"`
}

type FileOutputConfig struct {
	Directory *string `yaml:"directory,omitempty"`
}
type FileCacheConfig struct {
	ChunkPlans  *FileChunkPlanCacheConfig  `yaml:"chunk_plans,omitempty"`
	Checkpoints *FileCheckpointCacheConfig `yaml:"checkpoints,omitempty"`
}
type FileChunkPlanCacheConfig struct {
	Directory *string `yaml:"directory,omitempty"`
	Mode      *string `yaml:"mode,omitempty"`
}
type FileCheckpointCacheConfig struct {
	Enabled   *bool   `yaml:"enabled,omitempty"`
	Directory *string `yaml:"directory,omitempty"`
}
type FileDebugConfig struct {
	Directory *string `yaml:"directory,omitempty"`
}

type fileModuleBinding struct {
	Module     string
	LLMProfile string
	Retries    int
	Options    map[string]any
	References map[string]fileReferenceSource
	Validators pipeline.ValidatorOverride
}

type fileReferenceSource struct {
	path     string
	artifact *pipeline.ArtifactReference
}

func (source *fileReferenceSource) UnmarshalYAML(node *yaml.Node) error {
	if source == nil {
		return fmt.Errorf("reference source must not be nil")
	}
	switch node.Kind {
	case yaml.ScalarNode:
		if node.Tag != "!!str" {
			return fmt.Errorf("external reference path must be a string")
		}
		path := strings.TrimSpace(node.Value)
		if path == "" {
			return fmt.Errorf("external reference path must not be empty")
		}
		source.path = path
		source.artifact = nil
		return nil
	case yaml.MappingNode:
		if len(node.Content) != 2 || node.Content[0].Value != "artifact" {
			return fmt.Errorf("reference source mapping must contain only artifact")
		}
		artifactNode := node.Content[1]
		if artifactNode.Kind != yaml.MappingNode {
			return fmt.Errorf("artifact reference must be an object")
		}
		var step, lane string
		seen := map[string]bool{}
		for i := 0; i < len(artifactNode.Content); i += 2 {
			key := artifactNode.Content[i].Value
			value := artifactNode.Content[i+1]
			if seen[key] {
				return fmt.Errorf("artifact reference field %q is duplicated", key)
			}
			seen[key] = true
			if value.Tag != "!!str" {
				return fmt.Errorf("artifact reference field %q must be a string", key)
			}
			switch key {
			case "step":
				step = strings.TrimSpace(value.Value)
			case "lane":
				lane = strings.TrimSpace(value.Value)
			default:
				return fmt.Errorf("field %s not found in artifact reference", key)
			}
		}
		if step == "" || lane == "" {
			return fmt.Errorf("artifact reference step and lane must not be empty")
		}
		source.path = ""
		source.artifact = &pipeline.ArtifactReference{Step: step, Lane: lane}
		return nil
	default:
		return fmt.Errorf("reference source must be a string or object")
	}
}

func (source fileReferenceSource) toPipelineSource() pipeline.ReferenceSource {
	if source.artifact != nil {
		artifact := *source.artifact
		return pipeline.ReferenceSource{Artifact: &artifact}
	}
	return pipeline.ExternalReference(source.path)
}

func fileReferenceSourcesToPipeline(values map[string]fileReferenceSource) map[string]pipeline.ReferenceSource {
	if len(values) == 0 {
		return nil
	}
	out := make(map[string]pipeline.ReferenceSource, len(values))
	for key, value := range values {
		out[strings.TrimSpace(key)] = value.toPipelineSource()
	}
	return out
}

func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
	switch node.Kind {
	case yaml.ScalarNode:
		var module string
		if err := node.Decode(&module); err != nil {
			return fmt.Errorf("module binding must be a string or object")
		}
		b.Module = strings.TrimSpace(module)
		return nil
	case yaml.MappingNode:
		for i := 0; i < len(node.Content); i += 2 {
			keyNode := node.Content[i]
			valueNode := node.Content[i+1]
			switch keyNode.Value {
			case "module":
				var module string
				if err := valueNode.Decode(&module); err != nil {
					return err
				}
				b.Module = strings.TrimSpace(module)
			case "llm_profile":
				var llmProfile string
				if err := valueNode.Decode(&llmProfile); err != nil {
					return err
				}
				b.LLMProfile = strings.TrimSpace(llmProfile)
				if b.LLMProfile == "" {
					return fmt.Errorf("llm_profile must not be empty when set")
				}
			case "retries":
				var retries int
				if err := valueNode.Decode(&retries); err != nil {
					return err
				}
				b.Retries = retries
			case "options":
				var options map[string]any
				if err := valueNode.Decode(&options); err != nil {
					return err
				}
				b.Options = normalizeOptions(options)
			case "references":
				var references map[string]fileReferenceSource
				if err := valueNode.Decode(&references); err != nil {
					return err
				}
				b.References = references
			case "validators":
				b.Validators.Set = true
				var validators []fileModuleBinding
				if err := valueNode.Decode(&validators); err != nil {
					return err
				}
				b.Validators.Validators = make([]pipeline.ModuleBinding, len(validators))
				for i, validator := range validators {
					b.Validators.Validators[i] = validator.toPipelineBinding()
				}
			default:
				return fmt.Errorf("field %s not found in module binding", keyNode.Value)
			}
		}
		return nil
	default:
		return fmt.Errorf("module binding must be a string or object")
	}
}

func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
	return pipeline.ModuleBinding{
		Module:     strings.TrimSpace(b.Module),
		LLMProfile: strings.TrimSpace(b.LLMProfile),
		Retries:    b.Retries,
		Options:    cloneOptions(b.Options),
		References: fileReferenceSourcesToPipeline(b.References),
		Validators: b.Validators,
	}
}

func LoadFileConfig(path string) (FileConfig, error) {
	data, err := os.ReadFile(path)
	if err != nil {
		return FileConfig{}, fmt.Errorf("read config file %q: %w", path, err)
	}
	cfg, err := ParseFileConfigYAML(data)
	if err != nil {
		return FileConfig{}, fmt.Errorf("parse config file %q: %w", path, err)
	}
	return cfg, nil
}

func ParseFileConfigYAML(data []byte) (FileConfig, error) {
	var header struct {
		Version int `yaml:"version"`
	}
	if err := yaml.Unmarshal(data, &header); err != nil {
		return FileConfig{}, fmt.Errorf("decode yaml version header: %w", err)
	}
	if header.Version == 0 {
		return FileConfig{}, fmt.Errorf("config version is required")
	}
	if header.Version == 2 {
		return FileConfig{}, fmt.Errorf("config version 2 is no longer supported; migrate the file using the version 2-to-3 migration in docs/config.md")
	}
	if header.Version == 3 {
		return FileConfig{}, fmt.Errorf("config version 3 is no longer supported; change \"version: 3\" to \"version: 4\" and rename \"scriptorium:\" to \"promptkit:\"")
	}
	if header.Version != SupportedFileConfigVersion {
		return FileConfig{}, fmt.Errorf("unsupported config version %d (supported version is %d)", header.Version, SupportedFileConfigVersion)
	}
	var fileCfg FileConfig
	decoder := yaml.NewDecoder(bytes.NewReader(data))
	decoder.KnownFields(true)
	if err := decoder.Decode(&fileCfg); err != nil {
		return FileConfig{}, fmt.Errorf("decode yaml: %w", err)
	}
	var trailing any
	if err := decoder.Decode(&trailing); err == nil {
		return FileConfig{}, fmt.Errorf("config must contain exactly one YAML document")
	} else if err != io.EOF {
		return FileConfig{}, fmt.Errorf("decode trailing yaml document: %w", err)
	}
	return fileCfg, nil
}

func (c *Config) ApplyFileConfig(fileCfg FileConfig) error {
	return c.applyFileConfigWithLookup(fileCfg, os.LookupEnv)
}

func (c *Config) ApplyFileConfigWithLookup(fileCfg FileConfig, lookup func(string) (string, bool)) error {
	return c.applyFileConfigWithLookup(fileCfg, lookup)
}

func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(string) (string, bool)) error {
	_ = lookup
	if c == nil {
		return fmt.Errorf("config must not be nil")
	}
	if fileCfg.Version != SupportedFileConfigVersion {
		return fmt.Errorf("unsupported config version %d", fileCfg.Version)
	}
	if c.Pipelines == nil {
		c.Pipelines = map[string]pipeline.PipelineProfile{}
	}

	pipelineIDs, rawPipelineIDs, err := normalizedMapKeys(fileCfg.Pipelines, "pipeline id")
	if err != nil {
		return err
	}
	for _, pipelineID := range pipelineIDs {
		filePipeline := fileCfg.Pipelines[rawPipelineIDs[pipelineID]]
		hasArtifacts := filePipeline.artifactsSet || filePipeline.Artifacts != nil
		hasSteps := filePipeline.stepsSet || filePipeline.Steps != nil
		if hasArtifacts && hasSteps {
			return fmt.Errorf("pipeline %q must not declare both artifacts and steps", pipelineID)
		}
		if hasSteps && len(filePipeline.Steps) == 0 {
			return fmt.Errorf("pipeline %q must declare at least one ordered step", pipelineID)
		}
		if hasSteps {
			seenSteps := make(map[string]struct{}, len(filePipeline.Steps))
			seenLanes := make(map[string]struct{})
			for index, step := range filePipeline.Steps {
				stepID := strings.TrimSpace(step.ID)
				if stepID == "" {
					return fmt.Errorf("pipeline %q step[%d] id must not be empty", pipelineID, index)
				}
				if _, ok := seenSteps[stepID]; ok {
					return fmt.Errorf("pipeline %q step id %q is duplicated after trimming", pipelineID, stepID)
				}
				seenSteps[stepID] = struct{}{}
				laneIDs, rawLaneIDs, err := normalizedMapKeys(step.Artifacts, fmt.Sprintf("pipeline %q step %q artifact lane id", pipelineID, stepID))
				if err != nil {
					return err
				}
				for _, laneID := range laneIDs {
					if _, ok := seenLanes[laneID]; ok {
						return fmt.Errorf("pipeline %q artifact lane id %q is duplicated across steps", pipelineID, laneID)
					}
					seenLanes[laneID] = struct{}{}
					fileLane := step.Artifacts[rawLaneIDs[laneID]]
					if err := validateFileLaneReferences(pipelineID, stepID, laneID, fileLane); err != nil {
						return err
					}
				}
				if err := validateFileReferenceSources(step.References, fmt.Sprintf("pipeline %q step %q reference slot", pipelineID, stepID)); err != nil {
					return err
				}
			}
		}
		if _, _, err := normalizedMapKeys(filePipeline.Artifacts, fmt.Sprintf("pipeline %q artifact lane id", pipelineID)); err != nil {
			return err
		}
		if err := validateFileReferenceSources(filePipeline.References, fmt.Sprintf("pipeline %q reference slot", pipelineID)); err != nil {
			return err
		}
		if filePipeline.Chunk != nil {
			if _, _, err := normalizedMapKeys(filePipeline.Chunk.References, fmt.Sprintf("pipeline %q chunk reference slot", pipelineID)); err != nil {
				return err
			}
		}
		if _, _, err := normalizedMapKeys(filePipeline.Input.References, fmt.Sprintf("pipeline %q input reference slot", pipelineID)); err != nil {
			return err
		}
		if filePipeline.Output != nil {
			if _, _, err := normalizedMapKeys(filePipeline.Output.References, fmt.Sprintf("pipeline %q output reference slot", pipelineID)); err != nil {
				return err
			}
		}
		for rawLaneID, fileLane := range filePipeline.Artifacts {
			laneID := strings.TrimSpace(rawLaneID)
			if laneID == "" {
				continue
			}
			if _, _, err := normalizedMapKeys(fileLane.References, fmt.Sprintf("pipeline %q lane %q reference slot", pipelineID, laneID)); err != nil {
				return err
			}
			if _, _, err := normalizedMapKeys(fileLane.Extract.References, fmt.Sprintf("pipeline %q lane %q extract reference slot", pipelineID, laneID)); err != nil {
				return err
			}
			if fileLane.Merge != nil {
				if _, _, err := normalizedMapKeys(fileLane.Merge.References, fmt.Sprintf("pipeline %q lane %q merge reference slot", pipelineID, laneID)); err != nil {
					return err
				}
			}
			if fileLane.Normalize != nil {
				if _, _, err := normalizedMapKeys(fileLane.Normalize.References, fmt.Sprintf("pipeline %q lane %q normalize reference slot", pipelineID, laneID)); err != nil {
					return err
				}
			}
			for i, validator := range fileLane.Validators {
				if _, _, err := normalizedMapKeys(validator.References, fmt.Sprintf("pipeline %q lane %q validator[%d] reference slot", pipelineID, laneID, i)); err != nil {
					return err
				}
			}
		}
	}

	if fileCfg.PromptKit != nil {
		if fileCfg.PromptKit.ProfileDir != nil {
			value := strings.TrimSpace(*fileCfg.PromptKit.ProfileDir)
			if value == "" {
				return fmt.Errorf("promptkit.profile_dir must not be empty when set")
			}
			c.PromptKit.ProfileDir = value
		}
		if fileCfg.PromptKit.ProfileFile != nil {
			value := strings.TrimSpace(*fileCfg.PromptKit.ProfileFile)
			if value == "" {
				return fmt.Errorf("promptkit.profile_file must not be empty when set")
			}
			c.PromptKit.ProfileFile = value
		}
		if fileCfg.PromptKit.LocalBackend != nil {
			if fileCfg.PromptKit.LocalBackend.Endpoint == nil {
				return fmt.Errorf("promptkit.local_backend.endpoint must not be empty when set")
			}
			endpoint := strings.TrimSpace(*fileCfg.PromptKit.LocalBackend.Endpoint)
			if endpoint == "" {
				return fmt.Errorf("promptkit.local_backend.endpoint must not be empty when set")
			}
			localBackend := PromptKitLocalBackendConfig{Endpoint: endpoint}
			if fileCfg.PromptKit.LocalBackend.ConcurrencyLimit != nil {
				localBackend.ConcurrencyLimit = *fileCfg.PromptKit.LocalBackend.ConcurrencyLimit
			}
			c.PromptKit.LocalBackend = &localBackend
		}
	}

	for _, pipelineID := range pipelineIDs {
		filePipeline := fileCfg.Pipelines[rawPipelineIDs[pipelineID]]
		llmProfile := ""
		if filePipeline.llmProfileSet || filePipeline.LLMProfile != nil {
			if filePipeline.LLMProfile == nil || strings.TrimSpace(*filePipeline.LLMProfile) == "" {
				return fmt.Errorf("pipeline %q llm_profile must not be empty when set", pipelineID)
			}
			llmProfile = strings.TrimSpace(*filePipeline.LLMProfile)
		}
		hasSteps := filePipeline.stepsSet || filePipeline.Steps != nil
		laneIDs, rawLaneIDs, err := normalizedMapKeys(filePipeline.Artifacts, fmt.Sprintf("pipeline %q artifact lane id", pipelineID))
		if err != nil {
			return err
		}
		profile := pipeline.PipelineProfile{
			ID:         pipelineID,
			LLMProfile: llmProfile,
			Input:      filePipeline.Input.toPipelineBinding(),
			Artifacts:  make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
			References: fileReferenceSourcesToPipeline(filePipeline.References),
		}
		if filePipeline.Chunk != nil {
			profile.Chunk = filePipeline.Chunk.toPipelineBinding()
		}
		if filePipeline.Output != nil {
			profile.Output = filePipeline.Output.toPipelineBinding()
		}
		for _, laneID := range laneIDs {
			fileLane := filePipeline.Artifacts[rawLaneIDs[laneID]]
			extract := fileLane.Extract.toPipelineBinding()
			extract.References = mergeReferenceSources(fileReferenceSourcesToPipeline(fileLane.References), extract.References)
			lane := pipeline.ArtifactLaneProfile{
				Extract:    extract,
				References: fileReferenceSourcesToPipeline(fileLane.References),
			}
			if fileLane.Merge != nil {
				lane.Merge = fileLane.Merge.toPipelineBinding()
			}
			if fileLane.Normalize != nil {
				lane.Normalize = fileLane.Normalize.toPipelineBinding()
			}
			if len(fileLane.Validators) > 0 {
				lane.Validators = make([]pipeline.ModuleBinding, len(fileLane.Validators))
				for i, validator := range fileLane.Validators {
					lane.Validators[i] = validator.toPipelineBinding()
				}
			}
			profile.Artifacts[laneID] = lane
		}
		if hasSteps {
			profile.Artifacts = nil
			profile.Steps = make([]pipeline.PipelineStepProfile, len(filePipeline.Steps))
			for i, fileStep := range filePipeline.Steps {
				stepID := strings.TrimSpace(fileStep.ID)
				step := pipeline.PipelineStepProfile{
					ID:         stepID,
					Artifacts:  make(map[string]pipeline.ArtifactLaneProfile, len(fileStep.Artifacts)),
					References: fileReferenceSourcesToPipeline(fileStep.References),
				}
				stepLaneIDs, stepRawLaneIDs, err := normalizedMapKeys(fileStep.Artifacts, fmt.Sprintf("pipeline %q step %q artifact lane id", pipelineID, stepID))
				if err != nil {
					return err
				}
				for _, laneID := range stepLaneIDs {
					fileLane := fileStep.Artifacts[stepRawLaneIDs[laneID]]
					extract := fileLane.Extract.toPipelineBinding()
					extract.References = mergeReferenceSources(fileReferenceSourcesToPipeline(fileLane.References), extract.References)
					lane := pipeline.ArtifactLaneProfile{Extract: extract, References: fileReferenceSourcesToPipeline(fileLane.References)}
					if fileLane.Merge != nil {
						lane.Merge = fileLane.Merge.toPipelineBinding()
					}
					if fileLane.Normalize != nil {
						lane.Normalize = fileLane.Normalize.toPipelineBinding()
					}
					if len(fileLane.Validators) > 0 {
						lane.Validators = make([]pipeline.ModuleBinding, len(fileLane.Validators))
						for index, validator := range fileLane.Validators {
							lane.Validators[index] = validator.toPipelineBinding()
						}
					}
					step.Artifacts[laneID] = lane
				}
				profile.Steps[i] = step
			}
		}
		c.Pipelines[pipelineID] = profile
	}

	if fileCfg.Concurrency != nil && fileCfg.Concurrency.TotalLLM != nil {
		c.Concurrency.TotalLLM = *fileCfg.Concurrency.TotalLLM
	}
	if fileCfg.Concurrency != nil && fileCfg.Concurrency.StageWorkers != nil {
		workers, configured, err := normalizeStageWorkers(fileCfg.Concurrency.StageWorkers)
		if err != nil {
			return err
		}
		c.Concurrency.StageWorkers = workers
		c.Concurrency.extractWorkersConfigured = configured
	}
	c.Concurrency.recomputeStageWorkerDefaults()
	if fileCfg.Output != nil && fileCfg.Output.Directory != nil {
		c.Output.Directory = strings.TrimSpace(*fileCfg.Output.Directory)
		if c.Output.Directory == "" {
			return fmt.Errorf("output.directory must not be empty")
		}
		if strings.ContainsRune(c.Output.Directory, '\x00') {
			return fmt.Errorf("output.directory must not contain NUL")
		}
	}
	if fileCfg.Cache != nil {
		if fileCfg.Cache.ChunkPlans != nil {
			if fileCfg.Cache.ChunkPlans.Mode != nil {
				mode, err := pipeline.ParseChunkCacheMode(*fileCfg.Cache.ChunkPlans.Mode)
				if err != nil {
					return fmt.Errorf("cache.chunk_plans.mode: %w", err)
				}
				c.Cache.ChunkPlans.Mode = mode
			}
			if fileCfg.Cache.ChunkPlans.Directory != nil {
				c.Cache.ChunkPlans.Directory = cleanOptionalPath(*fileCfg.Cache.ChunkPlans.Directory)
				if strings.ContainsRune(c.Cache.ChunkPlans.Directory, '\x00') {
					return fmt.Errorf("cache.chunk_plans.directory must not contain NUL")
				}
			}
		}
		if fileCfg.Cache.Checkpoints != nil {
			if fileCfg.Cache.Checkpoints.Enabled != nil {
				c.Cache.Checkpoints.Enabled = *fileCfg.Cache.Checkpoints.Enabled
			}
			if fileCfg.Cache.Checkpoints.Directory != nil {
				c.Cache.Checkpoints.Directory = cleanOptionalPath(*fileCfg.Cache.Checkpoints.Directory)
				if strings.ContainsRune(c.Cache.Checkpoints.Directory, '\x00') {
					return fmt.Errorf("cache.checkpoints.directory must not contain NUL")
				}
			}
		}
	}
	if fileCfg.Debug != nil && fileCfg.Debug.Directory != nil {
		c.Debug.Directory = strings.TrimSpace(*fileCfg.Debug.Directory)
		if c.Debug.Directory == "" {
			return fmt.Errorf("debug.directory must not be empty")
		}
		if strings.ContainsRune(c.Debug.Directory, '\x00') {
			return fmt.Errorf("debug.directory must not contain NUL")
		}
	}

	return nil
}

func cleanOptionalPath(value string) string {
	value = strings.TrimSpace(value)
	if value == "" {
		return ""
	}
	return filepath.Clean(value)
}

func normalizeStageWorkers(values map[string]int) (map[string]int, bool, error) {
	workers := make(map[string]int, len(values))
	configured := false
	for rawKey, value := range values {
		key := strings.TrimSpace(rawKey)
		if key == "" {
			return nil, false, fmt.Errorf("concurrency.stage_workers key must not be empty")
		}
		if key != "extract" {
			return nil, false, fmt.Errorf("concurrency.stage_workers key %q is not supported", rawKey)
		}
		if _, exists := workers[key]; exists {
			return nil, false, fmt.Errorf("concurrency.stage_workers key %q is duplicated after trimming", key)
		}
		workers[key] = value
		configured = true
	}
	return workers, configured, nil
}

func normalizedMapKeys[T any](values map[string]T, keyName string) ([]string, map[string]string, error) {
	keys := make([]string, 0, len(values))
	rawByNormalized := make(map[string]string, len(values))
	for rawID := range values {
		id := strings.TrimSpace(rawID)
		if id == "" {
			return nil, nil, fmt.Errorf("%s must not be empty", keyName)
		}
		if _, ok := rawByNormalized[id]; ok {
			return nil, nil, fmt.Errorf("%s %q is duplicated after trimming", keyName, id)
		}
		rawByNormalized[id] = rawID
		keys = append(keys, id)
	}
	sort.Strings(keys)
	return keys, rawByNormalized, nil
}

func validateFileReferenceSources(values map[string]fileReferenceSource, context string) error {
	seen := make(map[string]struct{}, len(values))
	for rawSlot, source := range values {
		slot := strings.TrimSpace(rawSlot)
		if slot == "" {
			return fmt.Errorf("%s must not be empty", context)
		}
		if _, ok := seen[slot]; ok {
			return fmt.Errorf("%s %q is duplicated after trimming", context, slot)
		}
		seen[slot] = struct{}{}
		if source.artifact != nil {
			if strings.TrimSpace(source.artifact.Step) == "" || strings.TrimSpace(source.artifact.Lane) == "" {
				return fmt.Errorf("%s %q artifact selector step and lane must not be empty", context, slot)
			}
			if strings.TrimSpace(source.path) != "" {
				return fmt.Errorf("%s %q must contain either an external path or artifact selector", context, slot)
			}
			continue
		}
		if strings.TrimSpace(source.path) == "" {
			return fmt.Errorf("%s %q source must not be empty", context, slot)
		}
	}
	return nil
}

func validateFileLaneReferences(pipelineID, stepID, laneID string, lane FileArtifactLaneProfile) error {
	prefix := fmt.Sprintf("pipeline %q step %q lane %q", pipelineID, stepID, laneID)
	references := []struct {
		label  string
		values map[string]fileReferenceSource
	}{
		{label: "reference slot", values: lane.References},
		{label: "extract reference slot", values: lane.Extract.References},
	}
	if lane.Merge != nil {
		references = append(references, struct {
			label  string
			values map[string]fileReferenceSource
		}{label: "merge reference slot", values: lane.Merge.References})
	}
	if lane.Normalize != nil {
		references = append(references, struct {
			label  string
			values map[string]fileReferenceSource
		}{label: "normalize reference slot", values: lane.Normalize.References})
	}
	for _, item := range references {
		if err := validateFileReferenceSources(item.values, prefix+" "+item.label); err != nil {
			return err
		}
	}
	for index, validator := range lane.Validators {
		if err := validateFileReferenceSources(validator.References, fmt.Sprintf("%s validator[%d] reference slot", prefix, index)); err != nil {
			return err
		}
	}
	return nil
}

func mergeReferenceSources(base map[string]pipeline.ReferenceSource, override map[string]pipeline.ReferenceSource) map[string]pipeline.ReferenceSource {
	if len(base) == 0 && len(override) == 0 {
		return nil
	}
	out := make(map[string]pipeline.ReferenceSource, len(base)+len(override))
	for key, value := range base {
		out[key] = value
	}
	for key, value := range override {
		out[key] = value
	}
	return out
}

func normalizeOptions(options map[string]any) map[string]any {
	if len(options) == 0 {
		return nil
	}
	return cloneOptions(options)
}