Add config loading primitives

2026-07-03 16:51:41 +00:00
parent 145d1260f9
commit 8622d0eef8
10 changed files with 1041 additions and 0 deletions
--- a/internal/core/config/file_config.go
+++ b/internal/core/config/file_config.go
@@ -0,0 +1,308 @@
+package config
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"regexp"
+	"strings"
+	"time"
+
+	"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
+	"gopkg.in/yaml.v3"
+)
+
+var envVarNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
+
+type FileConfig struct {
+	Version     int                            `yaml:"version"`
+	LLMProfiles map[string]FileLLMProfile      `yaml:"llm_profiles,omitempty"`
+	Pipelines   map[string]FilePipelineProfile `yaml:"pipelines,omitempty"`
+	Concurrency *FileConcurrencyConfig         `yaml:"concurrency,omitempty"`
+	Diagnostics *FileDiagnosticsConfig         `yaml:"diagnostics,omitempty"`
+}
+
+type FileLLMProfile struct {
+	Provider       *string              `yaml:"provider,omitempty"`
+	BaseURL        *string              `yaml:"base_url,omitempty"`
+	Model          *string              `yaml:"model,omitempty"`
+	APIKeyEnv      *string              `yaml:"api_key_env,omitempty"`
+	Timeout        *fileDurationSeconds `yaml:"timeout,omitempty"`
+	MaxRetries     *int                 `yaml:"max_retries,omitempty"`
+	MaxConcurrency *int                 `yaml:"max_concurrency,omitempty"`
+}
+
+type FilePipelineProfile struct {
+	Input     fileModuleBinding                  `yaml:"input"`
+	Chunk     *fileModuleBinding                 `yaml:"chunk,omitempty"`
+	Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
+	Output    *fileModuleBinding                 `yaml:"output,omitempty"`
+}
+
+type FileArtifactLaneProfile struct {
+	Extract    fileModuleBinding   `yaml:"extract"`
+	Merge      *fileModuleBinding  `yaml:"merge,omitempty"`
+	Normalize  *fileModuleBinding  `yaml:"normalize,omitempty"`
+	Validators []fileModuleBinding `yaml:"validators,omitempty"`
+}
+
+type FileConcurrencyConfig struct {
+	TotalLLM *int `yaml:"total_llm,omitempty"`
+}
+
+type FileDiagnosticsConfig struct {
+	WorkDir   *string `yaml:"work_dir,omitempty"`
+	Retention *string `yaml:"retention,omitempty"`
+}
+
+type fileDurationSeconds struct {
+	seconds int
+}
+
+func (d *fileDurationSeconds) UnmarshalYAML(node *yaml.Node) error {
+	if node.Kind != yaml.ScalarNode {
+		return fmt.Errorf("must be an integer seconds value or duration string")
+	}
+	if node.Tag == "!!int" {
+		var seconds int
+		if err := node.Decode(&seconds); err != nil {
+			return fmt.Errorf("must be an integer seconds value or duration string")
+		}
+		d.seconds = seconds
+		return nil
+	}
+
+	var raw string
+	if err := node.Decode(&raw); err != nil {
+		return fmt.Errorf("must be an integer seconds value or duration string")
+	}
+	duration, err := time.ParseDuration(strings.TrimSpace(raw))
+	if err != nil {
+		return fmt.Errorf("invalid duration %q", raw)
+	}
+	if duration%time.Second != 0 {
+		return fmt.Errorf("duration %q must resolve to whole seconds", raw)
+	}
+	d.seconds = int(duration / time.Second)
+	return nil
+}
+
+func (d fileDurationSeconds) Seconds() int {
+	return d.seconds
+}
+
+type fileModuleBinding struct {
+	Module     string
+	LLMProfile string
+	Options    map[string]any
+}
+
+func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
+	switch node.Kind {
+	case yaml.ScalarNode:
+		var module string
+		if err := node.Decode(&module); err != nil {
+			return fmt.Errorf("module binding must be a string or object")
+		}
+		b.Module = strings.TrimSpace(module)
+		return nil
+	case yaml.MappingNode:
+		for i := 0; i < len(node.Content); i += 2 {
+			keyNode := node.Content[i]
+			valueNode := node.Content[i+1]
+			switch keyNode.Value {
+			case "module":
+				var module string
+				if err := valueNode.Decode(&module); err != nil {
+					return err
+				}
+				b.Module = strings.TrimSpace(module)
+			case "llm_profile":
+				var llmProfile string
+				if err := valueNode.Decode(&llmProfile); err != nil {
+					return err
+				}
+				b.LLMProfile = strings.TrimSpace(llmProfile)
+			case "options":
+				var options map[string]any
+				if err := valueNode.Decode(&options); err != nil {
+					return err
+				}
+				b.Options = normalizeOptions(options)
+			default:
+				return fmt.Errorf("field %s not found in module binding", keyNode.Value)
+			}
+		}
+		return nil
+	default:
+		return fmt.Errorf("module binding must be a string or object")
+	}
+}
+
+func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
+	return pipeline.ModuleBinding{
+		Module:     strings.TrimSpace(b.Module),
+		LLMProfile: strings.TrimSpace(b.LLMProfile),
+		Options:    cloneOptions(b.Options),
+	}
+}
+
+func LoadFileConfig(path string) (FileConfig, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return FileConfig{}, fmt.Errorf("read config file %q: %w", path, err)
+	}
+	cfg, err := ParseFileConfigYAML(data)
+	if err != nil {
+		return FileConfig{}, fmt.Errorf("parse config file %q: %w", path, err)
+	}
+	return cfg, nil
+}
+
+func ParseFileConfigYAML(data []byte) (FileConfig, error) {
+	var fileCfg FileConfig
+	decoder := yaml.NewDecoder(bytes.NewReader(data))
+	decoder.KnownFields(true)
+	if err := decoder.Decode(&fileCfg); err != nil {
+		return FileConfig{}, fmt.Errorf("decode yaml: %w", err)
+	}
+	if fileCfg.Version == 0 {
+		return FileConfig{}, fmt.Errorf("config version is required")
+	}
+	if fileCfg.Version != SupportedFileConfigVersion {
+		return FileConfig{}, fmt.Errorf("unsupported config version %d", fileCfg.Version)
+	}
+	return fileCfg, nil
+}
+
+func (c *Config) ApplyFileConfig(fileCfg FileConfig) error {
+	return c.applyFileConfigWithLookup(fileCfg, os.LookupEnv)
+}
+
+func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(string) (string, bool)) error {
+	if c == nil {
+		return fmt.Errorf("config must not be nil")
+	}
+	if fileCfg.Version != SupportedFileConfigVersion {
+		return fmt.Errorf("unsupported config version %d", fileCfg.Version)
+	}
+	if c.LLMProfiles == nil {
+		c.LLMProfiles = map[string]LLMProfile{}
+	}
+	if c.Pipelines == nil {
+		c.Pipelines = map[string]pipeline.PipelineProfile{}
+	}
+
+	for rawID, fileProfile := range fileCfg.LLMProfiles {
+		profileID := strings.TrimSpace(rawID)
+		if profileID == "" {
+			return fmt.Errorf("llm profile id must not be empty")
+		}
+		profile := c.LLMProfiles[profileID]
+		if fileProfile.Provider != nil {
+			profile.Provider = strings.TrimSpace(*fileProfile.Provider)
+		}
+		if fileProfile.BaseURL != nil {
+			profile.BaseURL = strings.TrimSpace(*fileProfile.BaseURL)
+		}
+		if fileProfile.Model != nil {
+			profile.Model = strings.TrimSpace(*fileProfile.Model)
+		}
+		if fileProfile.APIKeyEnv != nil {
+			apiKey, err := resolveAPIKeyEnv(*fileProfile.APIKeyEnv, lookup)
+			if err != nil {
+				return fmt.Errorf("llm_profiles.%s.api_key_env: %w", profileID, err)
+			}
+			profile.APIKeyEnv = strings.TrimSpace(*fileProfile.APIKeyEnv)
+			profile.APIKey = apiKey
+		}
+		if fileProfile.Timeout != nil {
+			profile.TimeoutSeconds = fileProfile.Timeout.Seconds()
+		}
+		if fileProfile.MaxRetries != nil {
+			profile.MaxRetries = *fileProfile.MaxRetries
+		}
+		if fileProfile.MaxConcurrency != nil {
+			profile.MaxConcurrency = *fileProfile.MaxConcurrency
+		}
+		c.LLMProfiles[profileID] = profile
+	}
+
+	for rawID, filePipeline := range fileCfg.Pipelines {
+		pipelineID := strings.TrimSpace(rawID)
+		if pipelineID == "" {
+			return fmt.Errorf("pipeline id must not be empty")
+		}
+		profile := pipeline.PipelineProfile{
+			ID:        pipelineID,
+			Input:     filePipeline.Input.toPipelineBinding(),
+			Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
+		}
+		if filePipeline.Chunk != nil {
+			profile.Chunk = filePipeline.Chunk.toPipelineBinding()
+		}
+		if filePipeline.Output != nil {
+			profile.Output = filePipeline.Output.toPipelineBinding()
+		}
+		for rawLaneID, fileLane := range filePipeline.Artifacts {
+			laneID := strings.TrimSpace(rawLaneID)
+			if laneID == "" {
+				return fmt.Errorf("pipeline %q artifact lane id must not be empty", pipelineID)
+			}
+			lane := pipeline.ArtifactLaneProfile{
+				Extract: fileLane.Extract.toPipelineBinding(),
+			}
+			if fileLane.Merge != nil {
+				lane.Merge = fileLane.Merge.toPipelineBinding()
+			}
+			if fileLane.Normalize != nil {
+				lane.Normalize = fileLane.Normalize.toPipelineBinding()
+			}
+			if len(fileLane.Validators) > 0 {
+				lane.Validators = make([]pipeline.ModuleBinding, len(fileLane.Validators))
+				for i, validator := range fileLane.Validators {
+					lane.Validators[i] = validator.toPipelineBinding()
+				}
+			}
+			profile.Artifacts[laneID] = lane
+		}
+		c.Pipelines[pipelineID] = profile
+	}
+
+	if fileCfg.Concurrency != nil && fileCfg.Concurrency.TotalLLM != nil {
+		c.Concurrency.TotalLLM = *fileCfg.Concurrency.TotalLLM
+	}
+	if fileCfg.Diagnostics != nil {
+		if fileCfg.Diagnostics.WorkDir != nil {
+			c.Diagnostics.WorkDir = strings.TrimSpace(*fileCfg.Diagnostics.WorkDir)
+		}
+		if fileCfg.Diagnostics.Retention != nil {
+			c.Diagnostics.Retention = diagnostics.RetentionMode(strings.TrimSpace(*fileCfg.Diagnostics.Retention))
+		}
+	}
+
+	return nil
+}
+
+func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) {
+	name := strings.TrimSpace(envName)
+	if name == "" {
+		return "", fmt.Errorf("must not be empty")
+	}
+	if !envVarNamePattern.MatchString(name) {
+		return "", fmt.Errorf("must be an environment variable name")
+	}
+	value, ok := lookup(name)
+	if !ok {
+		return "", fmt.Errorf("%s is not set", name)
+	}
+	return value, nil
+}
+
+func normalizeOptions(options map[string]any) map[string]any {
+	if len(options) == 0 {
+		return nil
+	}
+	return cloneOptions(options)
+}