package config import ( "bytes" "fmt" "os" "path/filepath" "sort" "strings" "gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gopkg.in/yaml.v3" ) type FileConfig struct { Version int `yaml:"version"` Scriptorium *FileScriptoriumConfig `yaml:"scriptorium,omitempty"` Pipelines map[string]FilePipelineProfile `yaml:"pipelines,omitempty"` Concurrency *FileConcurrencyConfig `yaml:"concurrency,omitempty"` Diagnostics *FileDiagnosticsConfig `yaml:"diagnostics,omitempty"` Workspace *FileWorkspaceConfig `yaml:"workspace,omitempty"` } type FileScriptoriumConfig struct { ProfileDir *string `yaml:"profile_dir,omitempty"` ProfileFile *string `yaml:"profile_file,omitempty"` } type FilePipelineProfile struct { Input fileModuleBinding `yaml:"input"` Chunk *fileModuleBinding `yaml:"chunk,omitempty"` Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"` Output *fileModuleBinding `yaml:"output,omitempty"` References map[string]string `yaml:"references,omitempty"` } type FileArtifactLaneProfile struct { Extract fileModuleBinding `yaml:"extract"` Merge *fileModuleBinding `yaml:"merge,omitempty"` Normalize *fileModuleBinding `yaml:"normalize,omitempty"` Validators []fileModuleBinding `yaml:"validators,omitempty"` References map[string]string `yaml:"references,omitempty"` } type FileConcurrencyConfig struct { TotalLLM *int `yaml:"total_llm,omitempty"` StageWorkers map[string]int `yaml:"stage_workers,omitempty"` } type FileDiagnosticsConfig struct { WorkDir *string `yaml:"work_dir,omitempty"` Retention *string `yaml:"retention,omitempty"` } type FileWorkspaceConfig struct { Directory *string `yaml:"directory,omitempty"` ChunkCache *FileWorkspaceChunkCacheConfig `yaml:"chunk_cache,omitempty"` Diagnostics *FileWorkspaceDiagnosticsConfig `yaml:"diagnostics,omitempty"` Resume *FileWorkspaceEnabledConfig `yaml:"resume,omitempty"` Debug *FileWorkspaceEnabledConfig `yaml:"debug,omitempty"` } type FileWorkspaceChunkCacheConfig struct { Mode *string `yaml:"mode,omitempty"` Directory *string `yaml:"directory,omitempty"` } type FileWorkspaceDiagnosticsConfig struct { Enabled *bool `yaml:"enabled,omitempty"` Retention *string `yaml:"retention,omitempty"` } type FileWorkspaceEnabledConfig struct { Enabled *bool `yaml:"enabled,omitempty"` } type fileModuleBinding struct { Module string LLMProfile string Retries int Options map[string]any References map[string]string Validators pipeline.ValidatorOverride } func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error { switch node.Kind { case yaml.ScalarNode: var module string if err := node.Decode(&module); err != nil { return fmt.Errorf("module binding must be a string or object") } b.Module = strings.TrimSpace(module) return nil case yaml.MappingNode: for i := 0; i < len(node.Content); i += 2 { keyNode := node.Content[i] valueNode := node.Content[i+1] switch keyNode.Value { case "module": var module string if err := valueNode.Decode(&module); err != nil { return err } b.Module = strings.TrimSpace(module) case "llm_profile": var llmProfile string if err := valueNode.Decode(&llmProfile); err != nil { return err } b.LLMProfile = strings.TrimSpace(llmProfile) case "retries": var retries int if err := valueNode.Decode(&retries); err != nil { return err } b.Retries = retries case "options": var options map[string]any if err := valueNode.Decode(&options); err != nil { return err } b.Options = normalizeOptions(options) case "references": var references map[string]string if err := valueNode.Decode(&references); err != nil { return err } b.References = references case "validators": b.Validators.Set = true var validators []fileModuleBinding if err := valueNode.Decode(&validators); err != nil { return err } b.Validators.Validators = make([]pipeline.ModuleBinding, len(validators)) for i, validator := range validators { b.Validators.Validators[i] = validator.toPipelineBinding() } default: return fmt.Errorf("field %s not found in module binding", keyNode.Value) } } return nil default: return fmt.Errorf("module binding must be a string or object") } } func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding { return pipeline.ModuleBinding{ Module: strings.TrimSpace(b.Module), LLMProfile: strings.TrimSpace(b.LLMProfile), Retries: b.Retries, Options: cloneOptions(b.Options), References: normalizedStringMap(b.References), Validators: b.Validators, } } func LoadFileConfig(path string) (FileConfig, error) { data, err := os.ReadFile(path) if err != nil { return FileConfig{}, fmt.Errorf("read config file %q: %w", path, err) } cfg, err := ParseFileConfigYAML(data) if err != nil { return FileConfig{}, fmt.Errorf("parse config file %q: %w", path, err) } return cfg, nil } func ParseFileConfigYAML(data []byte) (FileConfig, error) { var fileCfg FileConfig decoder := yaml.NewDecoder(bytes.NewReader(data)) decoder.KnownFields(true) if err := decoder.Decode(&fileCfg); err != nil { return FileConfig{}, fmt.Errorf("decode yaml: %w", err) } if fileCfg.Version == 0 { return FileConfig{}, fmt.Errorf("config version is required") } if fileCfg.Version != SupportedFileConfigVersion { return FileConfig{}, fmt.Errorf("unsupported config version %d", fileCfg.Version) } return fileCfg, nil } func (c *Config) ApplyFileConfig(fileCfg FileConfig) error { return c.applyFileConfigWithLookup(fileCfg, os.LookupEnv) } func (c *Config) ApplyFileConfigWithLookup(fileCfg FileConfig, lookup func(string) (string, bool)) error { return c.applyFileConfigWithLookup(fileCfg, lookup) } func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(string) (string, bool)) error { _ = lookup if c == nil { return fmt.Errorf("config must not be nil") } if fileCfg.Version != SupportedFileConfigVersion { return fmt.Errorf("unsupported config version %d", fileCfg.Version) } if c.Pipelines == nil { c.Pipelines = map[string]pipeline.PipelineProfile{} } pipelineIDs, rawPipelineIDs, err := normalizedMapKeys(fileCfg.Pipelines, "pipeline id") if err != nil { return err } for _, pipelineID := range pipelineIDs { filePipeline := fileCfg.Pipelines[rawPipelineIDs[pipelineID]] if _, _, err := normalizedMapKeys(filePipeline.Artifacts, fmt.Sprintf("pipeline %q artifact lane id", pipelineID)); err != nil { return err } if _, _, err := normalizedMapKeys(filePipeline.References, fmt.Sprintf("pipeline %q reference slot", pipelineID)); err != nil { return err } if filePipeline.Chunk != nil { if _, _, err := normalizedMapKeys(filePipeline.Chunk.References, fmt.Sprintf("pipeline %q chunk reference slot", pipelineID)); err != nil { return err } } if _, _, err := normalizedMapKeys(filePipeline.Input.References, fmt.Sprintf("pipeline %q input reference slot", pipelineID)); err != nil { return err } if filePipeline.Output != nil { if _, _, err := normalizedMapKeys(filePipeline.Output.References, fmt.Sprintf("pipeline %q output reference slot", pipelineID)); err != nil { return err } } for rawLaneID, fileLane := range filePipeline.Artifacts { laneID := strings.TrimSpace(rawLaneID) if laneID == "" { continue } if _, _, err := normalizedMapKeys(fileLane.References, fmt.Sprintf("pipeline %q lane %q reference slot", pipelineID, laneID)); err != nil { return err } if _, _, err := normalizedMapKeys(fileLane.Extract.References, fmt.Sprintf("pipeline %q lane %q extract reference slot", pipelineID, laneID)); err != nil { return err } if fileLane.Merge != nil { if _, _, err := normalizedMapKeys(fileLane.Merge.References, fmt.Sprintf("pipeline %q lane %q merge reference slot", pipelineID, laneID)); err != nil { return err } } if fileLane.Normalize != nil { if _, _, err := normalizedMapKeys(fileLane.Normalize.References, fmt.Sprintf("pipeline %q lane %q normalize reference slot", pipelineID, laneID)); err != nil { return err } } for i, validator := range fileLane.Validators { if _, _, err := normalizedMapKeys(validator.References, fmt.Sprintf("pipeline %q lane %q validator[%d] reference slot", pipelineID, laneID, i)); err != nil { return err } } } } if fileCfg.Scriptorium != nil { if fileCfg.Scriptorium.ProfileDir != nil { value := strings.TrimSpace(*fileCfg.Scriptorium.ProfileDir) if value == "" { return fmt.Errorf("scriptorium.profile_dir must not be empty when set") } c.Scriptorium.ProfileDir = value } if fileCfg.Scriptorium.ProfileFile != nil { value := strings.TrimSpace(*fileCfg.Scriptorium.ProfileFile) if value == "" { return fmt.Errorf("scriptorium.profile_file must not be empty when set") } c.Scriptorium.ProfileFile = value } } for _, pipelineID := range pipelineIDs { filePipeline := fileCfg.Pipelines[rawPipelineIDs[pipelineID]] laneIDs, rawLaneIDs, err := normalizedMapKeys(filePipeline.Artifacts, fmt.Sprintf("pipeline %q artifact lane id", pipelineID)) if err != nil { return err } profile := pipeline.PipelineProfile{ ID: pipelineID, Input: filePipeline.Input.toPipelineBinding(), Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)), References: normalizedStringMap(filePipeline.References), } if filePipeline.Chunk != nil { profile.Chunk = filePipeline.Chunk.toPipelineBinding() } if filePipeline.Output != nil { profile.Output = filePipeline.Output.toPipelineBinding() } for _, laneID := range laneIDs { fileLane := filePipeline.Artifacts[rawLaneIDs[laneID]] extract := fileLane.Extract.toPipelineBinding() extract.References = mergeStringMaps(normalizedStringMap(fileLane.References), extract.References) lane := pipeline.ArtifactLaneProfile{ Extract: extract, References: normalizedStringMap(fileLane.References), } if fileLane.Merge != nil { lane.Merge = fileLane.Merge.toPipelineBinding() } if fileLane.Normalize != nil { lane.Normalize = fileLane.Normalize.toPipelineBinding() } if len(fileLane.Validators) > 0 { lane.Validators = make([]pipeline.ModuleBinding, len(fileLane.Validators)) for i, validator := range fileLane.Validators { lane.Validators[i] = validator.toPipelineBinding() } } profile.Artifacts[laneID] = lane } c.Pipelines[pipelineID] = profile } if fileCfg.Concurrency != nil && fileCfg.Concurrency.TotalLLM != nil { c.Concurrency.TotalLLM = *fileCfg.Concurrency.TotalLLM } if fileCfg.Concurrency != nil && fileCfg.Concurrency.StageWorkers != nil { workers, configured, err := normalizeStageWorkers(fileCfg.Concurrency.StageWorkers) if err != nil { return err } c.Concurrency.StageWorkers = workers c.Concurrency.extractWorkersConfigured = configured } c.Concurrency.recomputeStageWorkerDefaults() if fileCfg.Diagnostics != nil { if fileCfg.Diagnostics.WorkDir != nil { c.Diagnostics.WorkDir = strings.TrimSpace(*fileCfg.Diagnostics.WorkDir) } if fileCfg.Diagnostics.Retention != nil { c.Diagnostics.Retention = diagnostics.RetentionMode(strings.TrimSpace(*fileCfg.Diagnostics.Retention)) } } if fileCfg.Workspace != nil { if fileCfg.Workspace.Directory != nil { c.Workspace.Directory = strings.TrimSpace(*fileCfg.Workspace.Directory) } if fileCfg.Workspace.ChunkCache != nil { if fileCfg.Workspace.ChunkCache.Mode != nil { mode, err := pipeline.ParseChunkCacheMode(*fileCfg.Workspace.ChunkCache.Mode) if err != nil { return fmt.Errorf("workspace.chunk_cache.mode: %w", err) } c.Workspace.ChunkCache.Mode = mode } if fileCfg.Workspace.ChunkCache.Directory != nil { c.Workspace.ChunkCache.Directory = cleanOptionalPath(*fileCfg.Workspace.ChunkCache.Directory) } } if fileCfg.Workspace.Diagnostics != nil { if fileCfg.Workspace.Diagnostics.Enabled != nil { c.Workspace.Diagnostics.Enabled = *fileCfg.Workspace.Diagnostics.Enabled c.Workspace.Diagnostics.enabledSet = true } if fileCfg.Workspace.Diagnostics.Retention != nil { c.Workspace.Diagnostics.Retention = diagnostics.RetentionMode(strings.TrimSpace(*fileCfg.Workspace.Diagnostics.Retention)) c.Workspace.Diagnostics.retentionSet = true } } if fileCfg.Workspace.Resume != nil && fileCfg.Workspace.Resume.Enabled != nil { c.Workspace.Resume.Enabled = *fileCfg.Workspace.Resume.Enabled } if fileCfg.Workspace.Debug != nil && fileCfg.Workspace.Debug.Enabled != nil { c.Workspace.Debug.Enabled = *fileCfg.Workspace.Debug.Enabled } } c.RecomputeEffectiveDiagnostics() return nil } func cleanOptionalPath(value string) string { value = strings.TrimSpace(value) if value == "" { return "" } return filepath.Clean(value) } func normalizeStageWorkers(values map[string]int) (map[string]int, bool, error) { workers := make(map[string]int, len(values)) configured := false for rawKey, value := range values { key := strings.TrimSpace(rawKey) if key == "" { return nil, false, fmt.Errorf("concurrency.stage_workers key must not be empty") } if key != "extract" { return nil, false, fmt.Errorf("concurrency.stage_workers key %q is not supported", rawKey) } if _, exists := workers[key]; exists { return nil, false, fmt.Errorf("concurrency.stage_workers key %q is duplicated after trimming", key) } workers[key] = value configured = true } return workers, configured, nil } func normalizedMapKeys[T any](values map[string]T, keyName string) ([]string, map[string]string, error) { keys := make([]string, 0, len(values)) rawByNormalized := make(map[string]string, len(values)) for rawID := range values { id := strings.TrimSpace(rawID) if id == "" { return nil, nil, fmt.Errorf("%s must not be empty", keyName) } if _, ok := rawByNormalized[id]; ok { return nil, nil, fmt.Errorf("%s %q is duplicated after trimming", keyName, id) } rawByNormalized[id] = rawID keys = append(keys, id) } sort.Strings(keys) return keys, rawByNormalized, nil } func normalizedStringMap(values map[string]string) map[string]string { if len(values) == 0 { return nil } out := make(map[string]string, len(values)) keys := make([]string, 0, len(values)) rawByNormalized := make(map[string]string, len(values)) for rawKey := range values { key := strings.TrimSpace(rawKey) rawByNormalized[key] = rawKey keys = append(keys, key) } sort.Strings(keys) for _, key := range keys { out[key] = strings.TrimSpace(values[rawByNormalized[key]]) } return out } func mergeStringMaps(base map[string]string, override map[string]string) map[string]string { if len(base) == 0 && len(override) == 0 { return nil } out := make(map[string]string, len(base)+len(override)) for key, value := range base { out[key] = value } for key, value := range override { out[key] = value } return out } func normalizeOptions(options map[string]any) map[string]any { if len(options) == 0 { return nil } return cloneOptions(options) }