package config import ( "bytes" "fmt" "os" "regexp" "sort" "strings" "time" "gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" "gopkg.in/yaml.v3" ) var envVarNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) type FileConfig struct { Version int `yaml:"version"` LLMProfiles map[string]FileLLMProfile `yaml:"llm_profiles,omitempty"` Pipelines map[string]FilePipelineProfile `yaml:"pipelines,omitempty"` Concurrency *FileConcurrencyConfig `yaml:"concurrency,omitempty"` Diagnostics *FileDiagnosticsConfig `yaml:"diagnostics,omitempty"` } type FileLLMProfile struct { Provider *string `yaml:"provider,omitempty"` BaseURL *string `yaml:"base_url,omitempty"` Model *string `yaml:"model,omitempty"` APIKeyEnv *string `yaml:"api_key_env,omitempty"` Timeout *fileDurationSeconds `yaml:"timeout,omitempty"` MaxRetries *int `yaml:"max_retries,omitempty"` MaxConcurrency *int `yaml:"max_concurrency,omitempty"` } type FilePipelineProfile struct { Input fileModuleBinding `yaml:"input"` Chunk *fileModuleBinding `yaml:"chunk,omitempty"` Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"` Output *fileModuleBinding `yaml:"output,omitempty"` } type FileArtifactLaneProfile struct { Extract fileModuleBinding `yaml:"extract"` Merge *fileModuleBinding `yaml:"merge,omitempty"` Normalize *fileModuleBinding `yaml:"normalize,omitempty"` Validators []fileModuleBinding `yaml:"validators,omitempty"` } type FileConcurrencyConfig struct { TotalLLM *int `yaml:"total_llm,omitempty"` } type FileDiagnosticsConfig struct { WorkDir *string `yaml:"work_dir,omitempty"` Retention *string `yaml:"retention,omitempty"` } type fileDurationSeconds struct { seconds int } func (d *fileDurationSeconds) UnmarshalYAML(node *yaml.Node) error { if node.Kind != yaml.ScalarNode { return fmt.Errorf("must be an integer seconds value or duration string") } if node.Tag == "!!int" { var seconds int if err := node.Decode(&seconds); err != nil { return fmt.Errorf("must be an integer seconds value or duration string") } d.seconds = seconds return nil } var raw string if err := node.Decode(&raw); err != nil { return fmt.Errorf("must be an integer seconds value or duration string") } duration, err := time.ParseDuration(strings.TrimSpace(raw)) if err != nil { return fmt.Errorf("invalid duration %q", raw) } if duration%time.Second != 0 { return fmt.Errorf("duration %q must resolve to whole seconds", raw) } d.seconds = int(duration / time.Second) return nil } func (d fileDurationSeconds) Seconds() int { return d.seconds } type fileModuleBinding struct { Module string LLMProfile string Options map[string]any } func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error { switch node.Kind { case yaml.ScalarNode: var module string if err := node.Decode(&module); err != nil { return fmt.Errorf("module binding must be a string or object") } b.Module = strings.TrimSpace(module) return nil case yaml.MappingNode: for i := 0; i < len(node.Content); i += 2 { keyNode := node.Content[i] valueNode := node.Content[i+1] switch keyNode.Value { case "module": var module string if err := valueNode.Decode(&module); err != nil { return err } b.Module = strings.TrimSpace(module) case "llm_profile": var llmProfile string if err := valueNode.Decode(&llmProfile); err != nil { return err } b.LLMProfile = strings.TrimSpace(llmProfile) case "options": var options map[string]any if err := valueNode.Decode(&options); err != nil { return err } b.Options = normalizeOptions(options) default: return fmt.Errorf("field %s not found in module binding", keyNode.Value) } } return nil default: return fmt.Errorf("module binding must be a string or object") } } func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding { return pipeline.ModuleBinding{ Module: strings.TrimSpace(b.Module), LLMProfile: strings.TrimSpace(b.LLMProfile), Options: cloneOptions(b.Options), } } func LoadFileConfig(path string) (FileConfig, error) { data, err := os.ReadFile(path) if err != nil { return FileConfig{}, fmt.Errorf("read config file %q: %w", path, err) } cfg, err := ParseFileConfigYAML(data) if err != nil { return FileConfig{}, fmt.Errorf("parse config file %q: %w", path, err) } return cfg, nil } func ParseFileConfigYAML(data []byte) (FileConfig, error) { var fileCfg FileConfig decoder := yaml.NewDecoder(bytes.NewReader(data)) decoder.KnownFields(true) if err := decoder.Decode(&fileCfg); err != nil { return FileConfig{}, fmt.Errorf("decode yaml: %w", err) } if fileCfg.Version == 0 { return FileConfig{}, fmt.Errorf("config version is required") } if fileCfg.Version != SupportedFileConfigVersion { return FileConfig{}, fmt.Errorf("unsupported config version %d", fileCfg.Version) } return fileCfg, nil } func (c *Config) ApplyFileConfig(fileCfg FileConfig) error { return c.applyFileConfigWithLookup(fileCfg, os.LookupEnv) } func (c *Config) ApplyFileConfigWithLookup(fileCfg FileConfig, lookup func(string) (string, bool)) error { return c.applyFileConfigWithLookup(fileCfg, lookup) } func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(string) (string, bool)) error { if c == nil { return fmt.Errorf("config must not be nil") } if fileCfg.Version != SupportedFileConfigVersion { return fmt.Errorf("unsupported config version %d", fileCfg.Version) } if c.LLMProfiles == nil { c.LLMProfiles = map[string]LLMProfile{} } if c.Pipelines == nil { c.Pipelines = map[string]pipeline.PipelineProfile{} } profileIDs, rawLLMProfileIDs, err := normalizedMapKeys(fileCfg.LLMProfiles, "llm profile id") if err != nil { return err } pipelineIDs, rawPipelineIDs, err := normalizedMapKeys(fileCfg.Pipelines, "pipeline id") if err != nil { return err } for _, pipelineID := range pipelineIDs { filePipeline := fileCfg.Pipelines[rawPipelineIDs[pipelineID]] if _, _, err := normalizedMapKeys(filePipeline.Artifacts, fmt.Sprintf("pipeline %q artifact lane id", pipelineID)); err != nil { return err } } for _, profileID := range profileIDs { fileProfile := fileCfg.LLMProfiles[rawLLMProfileIDs[profileID]] profile := c.LLMProfiles[profileID] if fileProfile.Provider != nil { profile.Provider = strings.TrimSpace(*fileProfile.Provider) } if fileProfile.BaseURL != nil { profile.BaseURL = strings.TrimSpace(*fileProfile.BaseURL) } if fileProfile.Model != nil { profile.Model = strings.TrimSpace(*fileProfile.Model) } if fileProfile.APIKeyEnv != nil { apiKey, err := resolveAPIKeyEnv(*fileProfile.APIKeyEnv, lookup) if err != nil { return fmt.Errorf("llm_profiles.%s.api_key_env: %w", profileID, err) } profile.APIKeyEnv = strings.TrimSpace(*fileProfile.APIKeyEnv) profile.APIKey = apiKey } if fileProfile.Timeout != nil { profile.TimeoutSeconds = fileProfile.Timeout.Seconds() } if fileProfile.MaxRetries != nil { profile.MaxRetries = *fileProfile.MaxRetries } if fileProfile.MaxConcurrency != nil { profile.MaxConcurrency = *fileProfile.MaxConcurrency } c.LLMProfiles[profileID] = profile } for _, pipelineID := range pipelineIDs { filePipeline := fileCfg.Pipelines[rawPipelineIDs[pipelineID]] laneIDs, rawLaneIDs, err := normalizedMapKeys(filePipeline.Artifacts, fmt.Sprintf("pipeline %q artifact lane id", pipelineID)) if err != nil { return err } profile := pipeline.PipelineProfile{ ID: pipelineID, Input: filePipeline.Input.toPipelineBinding(), Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)), } if filePipeline.Chunk != nil { profile.Chunk = filePipeline.Chunk.toPipelineBinding() } if filePipeline.Output != nil { profile.Output = filePipeline.Output.toPipelineBinding() } for _, laneID := range laneIDs { fileLane := filePipeline.Artifacts[rawLaneIDs[laneID]] lane := pipeline.ArtifactLaneProfile{ Extract: fileLane.Extract.toPipelineBinding(), } if fileLane.Merge != nil { lane.Merge = fileLane.Merge.toPipelineBinding() } if fileLane.Normalize != nil { lane.Normalize = fileLane.Normalize.toPipelineBinding() } if len(fileLane.Validators) > 0 { lane.Validators = make([]pipeline.ModuleBinding, len(fileLane.Validators)) for i, validator := range fileLane.Validators { lane.Validators[i] = validator.toPipelineBinding() } } profile.Artifacts[laneID] = lane } c.Pipelines[pipelineID] = profile } if fileCfg.Concurrency != nil && fileCfg.Concurrency.TotalLLM != nil { c.Concurrency.TotalLLM = *fileCfg.Concurrency.TotalLLM } if fileCfg.Diagnostics != nil { if fileCfg.Diagnostics.WorkDir != nil { c.Diagnostics.WorkDir = strings.TrimSpace(*fileCfg.Diagnostics.WorkDir) } if fileCfg.Diagnostics.Retention != nil { c.Diagnostics.Retention = diagnostics.RetentionMode(strings.TrimSpace(*fileCfg.Diagnostics.Retention)) } } return nil } func normalizedMapKeys[T any](values map[string]T, keyName string) ([]string, map[string]string, error) { keys := make([]string, 0, len(values)) rawByNormalized := make(map[string]string, len(values)) for rawID := range values { id := strings.TrimSpace(rawID) if id == "" { return nil, nil, fmt.Errorf("%s must not be empty", keyName) } if _, ok := rawByNormalized[id]; ok { return nil, nil, fmt.Errorf("%s %q is duplicated after trimming", keyName, id) } rawByNormalized[id] = rawID keys = append(keys, id) } sort.Strings(keys) return keys, rawByNormalized, nil } func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) { name := strings.TrimSpace(envName) if name == "" { return "", fmt.Errorf("must not be empty") } if !envVarNamePattern.MatchString(name) { return "", fmt.Errorf("must be an environment variable name") } value, ok := lookup(name) if !ok { return "", fmt.Errorf("%s is not set", name) } return value, nil } func normalizeOptions(options map[string]any) map[string]any { if len(options) == 0 { return nil } return cloneOptions(options) }