Add config loading primitives

2026-07-03 16:51:41 +00:00
parent 145d1260f9
commit 8622d0eef8
10 changed files with 1041 additions and 0 deletions
--- a/go.mod
+++ b/go.mod
@@ -1,3 +1,5 @@
 module gitea.maximumdirect.net/eric/notarius

 go 1.24.0
+
+require gopkg.in/yaml.v3 v3.0.1
--- a/go.sum
+++ b/go.sum
@@ -0,0 +1,4 @@
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
--- a/internal/core/config/config.go
+++ b/internal/core/config/config.go
@@ -0,0 +1,131 @@
+package config
+
+import (
+	"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
+)
+
+const SupportedFileConfigVersion = 1
+
+type Config struct {
+	LLMProfiles map[string]LLMProfile               `json:"llm_profiles"`
+	Pipelines   map[string]pipeline.PipelineProfile `json:"pipelines"`
+	Concurrency ConcurrencyConfig                   `json:"concurrency"`
+	Diagnostics DiagnosticsConfig                   `json:"diagnostics"`
+}
+
+type LLMProfile struct {
+	Provider       string `json:"provider,omitempty"`
+	BaseURL        string `json:"base_url,omitempty"`
+	Model          string `json:"model,omitempty"`
+	APIKey         string `json:"api_key,omitempty"`
+	APIKeyEnv      string `json:"api_key_env,omitempty"`
+	TimeoutSeconds int    `json:"timeout_seconds,omitempty"`
+	MaxRetries     int    `json:"max_retries,omitempty"`
+	MaxConcurrency int    `json:"max_concurrency,omitempty"`
+}
+
+type ConcurrencyConfig struct {
+	TotalLLM int `json:"total_llm"`
+}
+
+type DiagnosticsConfig struct {
+	WorkDir   string                    `json:"work_dir"`
+	Retention diagnostics.RetentionMode `json:"retention"`
+}
+
+func Default() Config {
+	return Config{
+		LLMProfiles: map[string]LLMProfile{
+			pipeline.DefaultLLMProfile: {
+				Provider:       "openai-compatible",
+				TimeoutSeconds: 600,
+				MaxRetries:     3,
+				MaxConcurrency: 1,
+			},
+		},
+		Pipelines: map[string]pipeline.PipelineProfile{},
+		Concurrency: ConcurrencyConfig{
+			TotalLLM: 1,
+		},
+		Diagnostics: DiagnosticsConfig{
+			WorkDir:   "/tmp/notarius",
+			Retention: diagnostics.RetentionAuto,
+		},
+	}
+}
+
+func cloneConfig(in Config) Config {
+	out := in
+	out.LLMProfiles = make(map[string]LLMProfile, len(in.LLMProfiles))
+	for key, profile := range in.LLMProfiles {
+		out.LLMProfiles[key] = profile
+	}
+	out.Pipelines = make(map[string]pipeline.PipelineProfile, len(in.Pipelines))
+	for key, profile := range in.Pipelines {
+		out.Pipelines[key] = clonePipelineProfile(profile)
+	}
+	return out
+}
+
+func clonePipelineProfile(in pipeline.PipelineProfile) pipeline.PipelineProfile {
+	out := in
+	out.Input = cloneModuleBinding(in.Input)
+	out.Chunk = cloneModuleBinding(in.Chunk)
+	out.Output = cloneModuleBinding(in.Output)
+	if len(in.Artifacts) > 0 {
+		out.Artifacts = make(map[string]pipeline.ArtifactLaneProfile, len(in.Artifacts))
+		for key, lane := range in.Artifacts {
+			out.Artifacts[key] = cloneArtifactLaneProfile(lane)
+		}
+	}
+	return out
+}
+
+func cloneArtifactLaneProfile(in pipeline.ArtifactLaneProfile) pipeline.ArtifactLaneProfile {
+	out := in
+	out.Extract = cloneModuleBinding(in.Extract)
+	out.Merge = cloneModuleBinding(in.Merge)
+	out.Normalize = cloneModuleBinding(in.Normalize)
+	if len(in.Validators) > 0 {
+		out.Validators = make([]pipeline.ModuleBinding, len(in.Validators))
+		for i, binding := range in.Validators {
+			out.Validators[i] = cloneModuleBinding(binding)
+		}
+	}
+	return out
+}
+
+func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
+	out := in
+	if len(in.Options) > 0 {
+		out.Options = cloneOptions(in.Options)
+	}
+	return out
+}
+
+func cloneOptions(in map[string]any) map[string]any {
+	if len(in) == 0 {
+		return nil
+	}
+	out := make(map[string]any, len(in))
+	for key, value := range in {
+		out[key] = cloneOptionValue(value)
+	}
+	return out
+}
+
+func cloneOptionValue(value any) any {
+	switch typed := value.(type) {
+	case map[string]any:
+		return cloneOptions(typed)
+	case []any:
+		out := make([]any, len(typed))
+		for i, item := range typed {
+			out[i] = cloneOptionValue(item)
+		}
+		return out
+	default:
+		return typed
+	}
+}
--- a/internal/core/config/config_test.go
+++ b/internal/core/config/config_test.go
@@ -0,0 +1,78 @@
+package config
+
+import (
+	"testing"
+
+	"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
+)
+
+func TestDefaultValues(t *testing.T) {
+	cfg := Default()
+
+	defaultProfile, ok := cfg.LLMProfiles[pipeline.DefaultLLMProfile]
+	if !ok {
+		t.Fatalf("expected default LLM profile")
+	}
+	if defaultProfile.Provider != "openai-compatible" {
+		t.Fatalf("unexpected provider: %q", defaultProfile.Provider)
+	}
+	if defaultProfile.BaseURL != "" || defaultProfile.Model != "" {
+		t.Fatalf("default profile should not require base URL/model yet: %+v", defaultProfile)
+	}
+	if defaultProfile.TimeoutSeconds != 600 || defaultProfile.MaxRetries != 3 || defaultProfile.MaxConcurrency != 1 {
+		t.Fatalf("unexpected default LLM operational values: %+v", defaultProfile)
+	}
+	if len(cfg.Pipelines) != 0 {
+		t.Fatalf("expected no built-in pipeline profiles, got %v", cfg.Pipelines)
+	}
+	if cfg.Concurrency.TotalLLM != 1 {
+		t.Fatalf("unexpected total LLM concurrency: %d", cfg.Concurrency.TotalLLM)
+	}
+	if cfg.Diagnostics.WorkDir != "/tmp/notarius" {
+		t.Fatalf("unexpected diagnostics work dir: %q", cfg.Diagnostics.WorkDir)
+	}
+	if cfg.Diagnostics.Retention != diagnostics.RetentionAuto {
+		t.Fatalf("unexpected diagnostics retention: %q", cfg.Diagnostics.Retention)
+	}
+}
+
+func TestApplyFileConfigMergesWithDefaults(t *testing.T) {
+	fileCfg, err := ParseFileConfigYAML([]byte(`
+version: 1
+llm_profiles:
+  default:
+    model: test-model
+pipelines:
+  example:
+    input: fake/input
+    artifacts:
+      events:
+        extract: fake/extract
+`))
+	if err != nil {
+		t.Fatalf("ParseFileConfigYAML: %v", err)
+	}
+
+	cfg := Default()
+	if err := cfg.applyFileConfigWithLookup(fileCfg, emptyLookup); err != nil {
+		t.Fatalf("ApplyFileConfig: %v", err)
+	}
+
+	profile := cfg.LLMProfiles[pipeline.DefaultLLMProfile]
+	if profile.Model != "test-model" {
+		t.Fatalf("expected file model, got %+v", profile)
+	}
+	if profile.Provider != "openai-compatible" || profile.TimeoutSeconds != 600 || profile.MaxRetries != 3 {
+		t.Fatalf("expected default LLM fields to be preserved, got %+v", profile)
+	}
+	if cfg.Concurrency.TotalLLM != 1 {
+		t.Fatalf("expected default concurrency preserved, got %d", cfg.Concurrency.TotalLLM)
+	}
+	if cfg.Diagnostics.Retention != diagnostics.RetentionAuto {
+		t.Fatalf("expected default diagnostics retention preserved, got %q", cfg.Diagnostics.Retention)
+	}
+	if _, ok := cfg.Pipelines["example"]; !ok {
+		t.Fatalf("expected file pipeline to be applied")
+	}
+}
--- a/internal/core/config/env.go
+++ b/internal/core/config/env.go
@@ -0,0 +1,88 @@
+package config
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+	"strings"
+
+	"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
+)
+
+func LoadFromEnv() (Config, error) {
+	cfg := Default()
+	if err := cfg.applyEnvOverridesWithLookup(os.LookupEnv); err != nil {
+		return Config{}, err
+	}
+	return cfg, nil
+}
+
+func (c *Config) ApplyEnvOverrides() error {
+	return c.applyEnvOverridesWithLookup(os.LookupEnv)
+}
+
+func (c *Config) applyEnvOverridesWithLookup(lookup func(string) (string, bool)) error {
+	if c == nil {
+		return fmt.Errorf("config must not be nil")
+	}
+	if c.LLMProfiles == nil {
+		c.LLMProfiles = map[string]LLMProfile{}
+	}
+
+	defaultProfile := c.LLMProfiles[pipeline.DefaultLLMProfile]
+	if raw, ok := lookup("NOTARIUS_LLM_DEFAULT_API_KEY"); ok {
+		defaultProfile.APIKey = raw
+	}
+	if raw, ok := lookup("NOTARIUS_LLM_DEFAULT_BASE_URL"); ok {
+		defaultProfile.BaseURL = strings.TrimSpace(raw)
+	}
+	if raw, ok := lookup("NOTARIUS_LLM_DEFAULT_MODEL"); ok {
+		defaultProfile.Model = strings.TrimSpace(raw)
+	}
+	if raw, ok := lookup("NOTARIUS_LLM_DEFAULT_TIMEOUT_SECONDS"); ok {
+		value, err := parseIntEnv("NOTARIUS_LLM_DEFAULT_TIMEOUT_SECONDS", raw)
+		if err != nil {
+			return err
+		}
+		defaultProfile.TimeoutSeconds = value
+	}
+	if raw, ok := lookup("NOTARIUS_LLM_DEFAULT_MAX_RETRIES"); ok {
+		value, err := parseIntEnv("NOTARIUS_LLM_DEFAULT_MAX_RETRIES", raw)
+		if err != nil {
+			return err
+		}
+		defaultProfile.MaxRetries = value
+	}
+	if raw, ok := lookup("NOTARIUS_LLM_DEFAULT_MAX_CONCURRENCY"); ok {
+		value, err := parseIntEnv("NOTARIUS_LLM_DEFAULT_MAX_CONCURRENCY", raw)
+		if err != nil {
+			return err
+		}
+		defaultProfile.MaxConcurrency = value
+	}
+	c.LLMProfiles[pipeline.DefaultLLMProfile] = defaultProfile
+
+	if raw, ok := lookup("NOTARIUS_TOTAL_LLM_CONCURRENCY"); ok {
+		value, err := parseIntEnv("NOTARIUS_TOTAL_LLM_CONCURRENCY", raw)
+		if err != nil {
+			return err
+		}
+		c.Concurrency.TotalLLM = value
+	}
+	if raw, ok := lookup("NOTARIUS_WORK_DIR"); ok {
+		c.Diagnostics.WorkDir = strings.TrimSpace(raw)
+	}
+	if raw, ok := lookup("NOTARIUS_DIAGNOSTICS_RETENTION"); ok {
+		c.Diagnostics.Retention = diagnostics.RetentionMode(strings.TrimSpace(raw))
+	}
+	return nil
+}
+
+func parseIntEnv(name string, raw string) (int, error) {
+	value, err := strconv.Atoi(strings.TrimSpace(raw))
+	if err != nil {
+		return 0, fmt.Errorf("%s: must be an integer", name)
+	}
+	return value, nil
+}
--- a/internal/core/config/env_test.go
+++ b/internal/core/config/env_test.go
@@ -0,0 +1,69 @@
+package config
+
+import (
+	"strings"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
+)
+
+func TestApplyEnvOverridesOperationalAndLLMValues(t *testing.T) {
+	cfg := Default()
+	cfg.Pipelines["example"] = pipeline.PipelineProfile{ID: "example", Input: pipeline.Binding("before")}
+
+	err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{
+		"NOTARIUS_LLM_DEFAULT_API_KEY":         "secret",
+		"NOTARIUS_LLM_DEFAULT_BASE_URL":        "https://example.invalid/v1",
+		"NOTARIUS_LLM_DEFAULT_MODEL":           "test-model",
+		"NOTARIUS_LLM_DEFAULT_TIMEOUT_SECONDS": "120",
+		"NOTARIUS_LLM_DEFAULT_MAX_RETRIES":     "5",
+		"NOTARIUS_LLM_DEFAULT_MAX_CONCURRENCY": "2",
+		"NOTARIUS_TOTAL_LLM_CONCURRENCY":       "3",
+		"NOTARIUS_WORK_DIR":                    "/tmp/notarius-env",
+		"NOTARIUS_DIAGNOSTICS_RETENTION":       "never",
+		"NOTARIUS_PIPELINE_INPUT":              "after",
+	}))
+	if err != nil {
+		t.Fatalf("ApplyEnvOverrides: %v", err)
+	}
+
+	profile := cfg.LLMProfiles[pipeline.DefaultLLMProfile]
+	if profile.APIKey != "secret" || profile.BaseURL != "https://example.invalid/v1" || profile.Model != "test-model" {
+		t.Fatalf("unexpected LLM profile strings: %+v", profile)
+	}
+	if profile.TimeoutSeconds != 120 || profile.MaxRetries != 5 || profile.MaxConcurrency != 2 {
+		t.Fatalf("unexpected LLM profile numeric values: %+v", profile)
+	}
+	if cfg.Concurrency.TotalLLM != 3 {
+		t.Fatalf("unexpected total concurrency: %d", cfg.Concurrency.TotalLLM)
+	}
+	if cfg.Diagnostics.WorkDir != "/tmp/notarius-env" || cfg.Diagnostics.Retention != diagnostics.RetentionNever {
+		t.Fatalf("unexpected diagnostics config: %+v", cfg.Diagnostics)
+	}
+	if cfg.Pipelines["example"].Input.Module != "before" {
+		t.Fatalf("environment overrides must not change pipeline wiring: %+v", cfg.Pipelines["example"])
+	}
+}
+
+func TestApplyEnvOverridesRejectsInvalidIntegers(t *testing.T) {
+	cfg := Default()
+	err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{
+		"NOTARIUS_TOTAL_LLM_CONCURRENCY": "many",
+	}))
+	if err == nil || !strings.Contains(err.Error(), "NOTARIUS_TOTAL_LLM_CONCURRENCY") {
+		t.Fatalf("expected named integer error, got %v", err)
+	}
+}
+
+func TestLoadFromEnvUsesDefaultConfig(t *testing.T) {
+	t.Setenv("NOTARIUS_LLM_DEFAULT_MODEL", "env-model")
+
+	cfg, err := LoadFromEnv()
+	if err != nil {
+		t.Fatalf("LoadFromEnv: %v", err)
+	}
+	if cfg.LLMProfiles[pipeline.DefaultLLMProfile].Model != "env-model" {
+		t.Fatalf("expected env model, got %+v", cfg.LLMProfiles[pipeline.DefaultLLMProfile])
+	}
+}
--- a/internal/core/config/file_config.go
+++ b/internal/core/config/file_config.go
@@ -0,0 +1,308 @@
+package config
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"regexp"
+	"strings"
+	"time"
+
+	"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
+	"gopkg.in/yaml.v3"
+)
+
+var envVarNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`)
+
+type FileConfig struct {
+	Version     int                            `yaml:"version"`
+	LLMProfiles map[string]FileLLMProfile      `yaml:"llm_profiles,omitempty"`
+	Pipelines   map[string]FilePipelineProfile `yaml:"pipelines,omitempty"`
+	Concurrency *FileConcurrencyConfig         `yaml:"concurrency,omitempty"`
+	Diagnostics *FileDiagnosticsConfig         `yaml:"diagnostics,omitempty"`
+}
+
+type FileLLMProfile struct {
+	Provider       *string              `yaml:"provider,omitempty"`
+	BaseURL        *string              `yaml:"base_url,omitempty"`
+	Model          *string              `yaml:"model,omitempty"`
+	APIKeyEnv      *string              `yaml:"api_key_env,omitempty"`
+	Timeout        *fileDurationSeconds `yaml:"timeout,omitempty"`
+	MaxRetries     *int                 `yaml:"max_retries,omitempty"`
+	MaxConcurrency *int                 `yaml:"max_concurrency,omitempty"`
+}
+
+type FilePipelineProfile struct {
+	Input     fileModuleBinding                  `yaml:"input"`
+	Chunk     *fileModuleBinding                 `yaml:"chunk,omitempty"`
+	Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
+	Output    *fileModuleBinding                 `yaml:"output,omitempty"`
+}
+
+type FileArtifactLaneProfile struct {
+	Extract    fileModuleBinding   `yaml:"extract"`
+	Merge      *fileModuleBinding  `yaml:"merge,omitempty"`
+	Normalize  *fileModuleBinding  `yaml:"normalize,omitempty"`
+	Validators []fileModuleBinding `yaml:"validators,omitempty"`
+}
+
+type FileConcurrencyConfig struct {
+	TotalLLM *int `yaml:"total_llm,omitempty"`
+}
+
+type FileDiagnosticsConfig struct {
+	WorkDir   *string `yaml:"work_dir,omitempty"`
+	Retention *string `yaml:"retention,omitempty"`
+}
+
+type fileDurationSeconds struct {
+	seconds int
+}
+
+func (d *fileDurationSeconds) UnmarshalYAML(node *yaml.Node) error {
+	if node.Kind != yaml.ScalarNode {
+		return fmt.Errorf("must be an integer seconds value or duration string")
+	}
+	if node.Tag == "!!int" {
+		var seconds int
+		if err := node.Decode(&seconds); err != nil {
+			return fmt.Errorf("must be an integer seconds value or duration string")
+		}
+		d.seconds = seconds
+		return nil
+	}
+
+	var raw string
+	if err := node.Decode(&raw); err != nil {
+		return fmt.Errorf("must be an integer seconds value or duration string")
+	}
+	duration, err := time.ParseDuration(strings.TrimSpace(raw))
+	if err != nil {
+		return fmt.Errorf("invalid duration %q", raw)
+	}
+	if duration%time.Second != 0 {
+		return fmt.Errorf("duration %q must resolve to whole seconds", raw)
+	}
+	d.seconds = int(duration / time.Second)
+	return nil
+}
+
+func (d fileDurationSeconds) Seconds() int {
+	return d.seconds
+}
+
+type fileModuleBinding struct {
+	Module     string
+	LLMProfile string
+	Options    map[string]any
+}
+
+func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
+	switch node.Kind {
+	case yaml.ScalarNode:
+		var module string
+		if err := node.Decode(&module); err != nil {
+			return fmt.Errorf("module binding must be a string or object")
+		}
+		b.Module = strings.TrimSpace(module)
+		return nil
+	case yaml.MappingNode:
+		for i := 0; i < len(node.Content); i += 2 {
+			keyNode := node.Content[i]
+			valueNode := node.Content[i+1]
+			switch keyNode.Value {
+			case "module":
+				var module string
+				if err := valueNode.Decode(&module); err != nil {
+					return err
+				}
+				b.Module = strings.TrimSpace(module)
+			case "llm_profile":
+				var llmProfile string
+				if err := valueNode.Decode(&llmProfile); err != nil {
+					return err
+				}
+				b.LLMProfile = strings.TrimSpace(llmProfile)
+			case "options":
+				var options map[string]any
+				if err := valueNode.Decode(&options); err != nil {
+					return err
+				}
+				b.Options = normalizeOptions(options)
+			default:
+				return fmt.Errorf("field %s not found in module binding", keyNode.Value)
+			}
+		}
+		return nil
+	default:
+		return fmt.Errorf("module binding must be a string or object")
+	}
+}
+
+func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
+	return pipeline.ModuleBinding{
+		Module:     strings.TrimSpace(b.Module),
+		LLMProfile: strings.TrimSpace(b.LLMProfile),
+		Options:    cloneOptions(b.Options),
+	}
+}
+
+func LoadFileConfig(path string) (FileConfig, error) {
+	data, err := os.ReadFile(path)
+	if err != nil {
+		return FileConfig{}, fmt.Errorf("read config file %q: %w", path, err)
+	}
+	cfg, err := ParseFileConfigYAML(data)
+	if err != nil {
+		return FileConfig{}, fmt.Errorf("parse config file %q: %w", path, err)
+	}
+	return cfg, nil
+}
+
+func ParseFileConfigYAML(data []byte) (FileConfig, error) {
+	var fileCfg FileConfig
+	decoder := yaml.NewDecoder(bytes.NewReader(data))
+	decoder.KnownFields(true)
+	if err := decoder.Decode(&fileCfg); err != nil {
+		return FileConfig{}, fmt.Errorf("decode yaml: %w", err)
+	}
+	if fileCfg.Version == 0 {
+		return FileConfig{}, fmt.Errorf("config version is required")
+	}
+	if fileCfg.Version != SupportedFileConfigVersion {
+		return FileConfig{}, fmt.Errorf("unsupported config version %d", fileCfg.Version)
+	}
+	return fileCfg, nil
+}
+
+func (c *Config) ApplyFileConfig(fileCfg FileConfig) error {
+	return c.applyFileConfigWithLookup(fileCfg, os.LookupEnv)
+}
+
+func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(string) (string, bool)) error {
+	if c == nil {
+		return fmt.Errorf("config must not be nil")
+	}
+	if fileCfg.Version != SupportedFileConfigVersion {
+		return fmt.Errorf("unsupported config version %d", fileCfg.Version)
+	}
+	if c.LLMProfiles == nil {
+		c.LLMProfiles = map[string]LLMProfile{}
+	}
+	if c.Pipelines == nil {
+		c.Pipelines = map[string]pipeline.PipelineProfile{}
+	}
+
+	for rawID, fileProfile := range fileCfg.LLMProfiles {
+		profileID := strings.TrimSpace(rawID)
+		if profileID == "" {
+			return fmt.Errorf("llm profile id must not be empty")
+		}
+		profile := c.LLMProfiles[profileID]
+		if fileProfile.Provider != nil {
+			profile.Provider = strings.TrimSpace(*fileProfile.Provider)
+		}
+		if fileProfile.BaseURL != nil {
+			profile.BaseURL = strings.TrimSpace(*fileProfile.BaseURL)
+		}
+		if fileProfile.Model != nil {
+			profile.Model = strings.TrimSpace(*fileProfile.Model)
+		}
+		if fileProfile.APIKeyEnv != nil {
+			apiKey, err := resolveAPIKeyEnv(*fileProfile.APIKeyEnv, lookup)
+			if err != nil {
+				return fmt.Errorf("llm_profiles.%s.api_key_env: %w", profileID, err)
+			}
+			profile.APIKeyEnv = strings.TrimSpace(*fileProfile.APIKeyEnv)
+			profile.APIKey = apiKey
+		}
+		if fileProfile.Timeout != nil {
+			profile.TimeoutSeconds = fileProfile.Timeout.Seconds()
+		}
+		if fileProfile.MaxRetries != nil {
+			profile.MaxRetries = *fileProfile.MaxRetries
+		}
+		if fileProfile.MaxConcurrency != nil {
+			profile.MaxConcurrency = *fileProfile.MaxConcurrency
+		}
+		c.LLMProfiles[profileID] = profile
+	}
+
+	for rawID, filePipeline := range fileCfg.Pipelines {
+		pipelineID := strings.TrimSpace(rawID)
+		if pipelineID == "" {
+			return fmt.Errorf("pipeline id must not be empty")
+		}
+		profile := pipeline.PipelineProfile{
+			ID:        pipelineID,
+			Input:     filePipeline.Input.toPipelineBinding(),
+			Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
+		}
+		if filePipeline.Chunk != nil {
+			profile.Chunk = filePipeline.Chunk.toPipelineBinding()
+		}
+		if filePipeline.Output != nil {
+			profile.Output = filePipeline.Output.toPipelineBinding()
+		}
+		for rawLaneID, fileLane := range filePipeline.Artifacts {
+			laneID := strings.TrimSpace(rawLaneID)
+			if laneID == "" {
+				return fmt.Errorf("pipeline %q artifact lane id must not be empty", pipelineID)
+			}
+			lane := pipeline.ArtifactLaneProfile{
+				Extract: fileLane.Extract.toPipelineBinding(),
+			}
+			if fileLane.Merge != nil {
+				lane.Merge = fileLane.Merge.toPipelineBinding()
+			}
+			if fileLane.Normalize != nil {
+				lane.Normalize = fileLane.Normalize.toPipelineBinding()
+			}
+			if len(fileLane.Validators) > 0 {
+				lane.Validators = make([]pipeline.ModuleBinding, len(fileLane.Validators))
+				for i, validator := range fileLane.Validators {
+					lane.Validators[i] = validator.toPipelineBinding()
+				}
+			}
+			profile.Artifacts[laneID] = lane
+		}
+		c.Pipelines[pipelineID] = profile
+	}
+
+	if fileCfg.Concurrency != nil && fileCfg.Concurrency.TotalLLM != nil {
+		c.Concurrency.TotalLLM = *fileCfg.Concurrency.TotalLLM
+	}
+	if fileCfg.Diagnostics != nil {
+		if fileCfg.Diagnostics.WorkDir != nil {
+			c.Diagnostics.WorkDir = strings.TrimSpace(*fileCfg.Diagnostics.WorkDir)
+		}
+		if fileCfg.Diagnostics.Retention != nil {
+			c.Diagnostics.Retention = diagnostics.RetentionMode(strings.TrimSpace(*fileCfg.Diagnostics.Retention))
+		}
+	}
+
+	return nil
+}
+
+func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) {
+	name := strings.TrimSpace(envName)
+	if name == "" {
+		return "", fmt.Errorf("must not be empty")
+	}
+	if !envVarNamePattern.MatchString(name) {
+		return "", fmt.Errorf("must be an environment variable name")
+	}
+	value, ok := lookup(name)
+	if !ok {
+		return "", fmt.Errorf("%s is not set", name)
+	}
+	return value, nil
+}
+
+func normalizeOptions(options map[string]any) map[string]any {
+	if len(options) == 0 {
+		return nil
+	}
+	return cloneOptions(options)
+}
--- a/internal/core/config/file_config_test.go
+++ b/internal/core/config/file_config_test.go
@@ -0,0 +1,310 @@
+package config
+
+import (
+	"os"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"testing"
+
+	"gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics"
+)
+
+func TestParseMinimalValidConfig(t *testing.T) {
+	fileCfg, err := ParseFileConfigYAML([]byte(`
+version: 1
+`))
+	if err != nil {
+		t.Fatalf("ParseFileConfigYAML: %v", err)
+	}
+	if fileCfg.Version != SupportedFileConfigVersion {
+		t.Fatalf("unexpected version: %d", fileCfg.Version)
+	}
+}
+
+func TestLoadFileConfig(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "config.yml")
+	if err := os.WriteFile(path, []byte("version: 1\n"), 0o644); err != nil {
+		t.Fatalf("write config: %v", err)
+	}
+
+	fileCfg, err := LoadFileConfig(path)
+	if err != nil {
+		t.Fatalf("LoadFileConfig: %v", err)
+	}
+	if fileCfg.Version != SupportedFileConfigVersion {
+		t.Fatalf("unexpected version: %d", fileCfg.Version)
+	}
+}
+
+func TestParseFileConfigRejectsUnknownYAMLFields(t *testing.T) {
+	_, err := ParseFileConfigYAML([]byte(`
+version: 1
+unexpected: true
+`))
+	if err == nil || !strings.Contains(err.Error(), "field unexpected not found") {
+		t.Fatalf("expected unknown field error, got %v", err)
+	}
+}
+
+func TestParseFileConfigRejectsUnknownModuleBindingFields(t *testing.T) {
+	_, err := ParseFileConfigYAML([]byte(`
+version: 1
+pipelines:
+  example:
+    input:
+      module: fake/input
+      unexpected: true
+    artifacts:
+      events:
+        extract: fake/extract
+`))
+	if err == nil || !strings.Contains(err.Error(), "field unexpected not found") {
+		t.Fatalf("expected unknown binding field error, got %v", err)
+	}
+}
+
+func TestParseFileConfigRejectsMissingAndUnsupportedVersion(t *testing.T) {
+	tests := []struct {
+		name string
+		data string
+		want string
+	}{
+		{name: "missing", data: `llm_profiles: {}`, want: "version is required"},
+		{name: "unsupported", data: `version: 2`, want: "unsupported config version"},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			_, err := ParseFileConfigYAML([]byte(tc.data))
+			if err == nil || !strings.Contains(err.Error(), tc.want) {
+				t.Fatalf("expected error containing %q, got %v", tc.want, err)
+			}
+		})
+	}
+}
+
+func TestParseFileConfigModuleBindingForms(t *testing.T) {
+	cfg := parseAndApplyConfig(t, `
+version: 1
+pipelines:
+  example:
+    input: fake/input
+    chunk:
+      module: generic
+      options:
+        size: 10
+        flags:
+          - alpha
+        nested:
+          enabled: true
+    artifacts:
+      events:
+        extract:
+          module: fake/extract
+          llm_profile: fast
+          options:
+            temperature: 0
+        merge: appendorder
+        normalize:
+          module: noop
+    output: json
+`)
+
+	profile := cfg.Pipelines["example"]
+	if profile.Input.Module != "fake/input" {
+		t.Fatalf("unexpected input binding: %+v", profile.Input)
+	}
+	if profile.Chunk.Module != "generic" {
+		t.Fatalf("unexpected chunk binding: %+v", profile.Chunk)
+	}
+	if profile.Chunk.Options["size"] != 10 {
+		t.Fatalf("expected chunk options to preserve scalar, got %#v", profile.Chunk.Options)
+	}
+	if !reflect.DeepEqual(profile.Chunk.Options["flags"], []any{"alpha"}) {
+		t.Fatalf("expected list option, got %#v", profile.Chunk.Options["flags"])
+	}
+	nested, ok := profile.Chunk.Options["nested"].(map[string]any)
+	if !ok || nested["enabled"] != true {
+		t.Fatalf("expected nested map option, got %#v", profile.Chunk.Options["nested"])
+	}
+
+	lane := profile.Artifacts["events"]
+	if lane.Extract.Module != "fake/extract" || lane.Extract.LLMProfile != "fast" {
+		t.Fatalf("unexpected extract binding: %+v", lane.Extract)
+	}
+	if lane.Extract.Options["temperature"] != 0 {
+		t.Fatalf("expected object options, got %#v", lane.Extract.Options)
+	}
+	if lane.Merge.Module != "appendorder" || lane.Normalize.Module != "noop" {
+		t.Fatalf("unexpected lane defaults: %+v", lane)
+	}
+	if profile.Output.Module != "json" {
+		t.Fatalf("unexpected output binding: %+v", profile.Output)
+	}
+}
+
+func TestParseFileConfigValidatorMixedBindingForms(t *testing.T) {
+	cfg := parseAndApplyConfig(t, `
+version: 1
+pipelines:
+  example:
+    input: fake/input
+    artifacts:
+      events:
+        extract: fake/extract
+        validators:
+          - fake/validator
+          - module: fake/llm-validator
+            llm_profile: careful
+            options:
+              threshold: 0.7
+`)
+
+	validators := cfg.Pipelines["example"].Artifacts["events"].Validators
+	if len(validators) != 2 {
+		t.Fatalf("expected two validators, got %d", len(validators))
+	}
+	if validators[0].Module != "fake/validator" {
+		t.Fatalf("unexpected shorthand validator: %+v", validators[0])
+	}
+	if validators[1].Module != "fake/llm-validator" || validators[1].LLMProfile != "careful" {
+		t.Fatalf("unexpected object validator: %+v", validators[1])
+	}
+	if validators[1].Options["threshold"] != 0.7 {
+		t.Fatalf("unexpected validator options: %#v", validators[1].Options)
+	}
+}
+
+func TestParseFileConfigDurationParsing(t *testing.T) {
+	tests := []struct {
+		name string
+		raw  string
+		want int
+	}{
+		{name: "integer seconds", raw: "600", want: 600},
+		{name: "duration string", raw: "10m", want: 600},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			cfg := parseAndApplyConfig(t, `
+version: 1
+llm_profiles:
+  default:
+    timeout: `+tc.raw+`
+`)
+			if got := cfg.LLMProfiles["default"].TimeoutSeconds; got != tc.want {
+				t.Fatalf("TimeoutSeconds = %d, want %d", got, tc.want)
+			}
+		})
+	}
+}
+
+func TestParseFileConfigRejectsSubsecondDuration(t *testing.T) {
+	_, err := ParseFileConfigYAML([]byte(`
+version: 1
+llm_profiles:
+  default:
+    timeout: 1500ms
+`))
+	if err == nil || !strings.Contains(err.Error(), "whole seconds") {
+		t.Fatalf("expected whole-seconds duration error, got %v", err)
+	}
+}
+
+func TestApplyFileConfigResolvesAPIKeyEnv(t *testing.T) {
+	fileCfg, err := ParseFileConfigYAML([]byte(`
+version: 1
+llm_profiles:
+  default:
+    api_key_env: NOTARIUS_TEST_API_KEY
+`))
+	if err != nil {
+		t.Fatalf("ParseFileConfigYAML: %v", err)
+	}
+
+	cfg := Default()
+	if err := cfg.applyFileConfigWithLookup(fileCfg, mapLookup(map[string]string{"NOTARIUS_TEST_API_KEY": "secret"})); err != nil {
+		t.Fatalf("ApplyFileConfig: %v", err)
+	}
+	profile := cfg.LLMProfiles["default"]
+	if profile.APIKeyEnv != "NOTARIUS_TEST_API_KEY" || profile.APIKey != "secret" {
+		t.Fatalf("unexpected resolved API key: %+v", profile)
+	}
+}
+
+func TestApplyFileConfigRejectsInvalidAPIKeyEnv(t *testing.T) {
+	tests := []struct {
+		name string
+		env  string
+		want string
+	}{
+		{name: "invalid name", env: "NOTARIUS-KEY", want: "environment variable name"},
+		{name: "not set", env: "NOTARIUS_TEST_API_KEY", want: "is not set"},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			fileCfg, err := ParseFileConfigYAML([]byte(`
+version: 1
+llm_profiles:
+  default:
+    api_key_env: ` + tc.env + `
+`))
+			if err != nil {
+				t.Fatalf("ParseFileConfigYAML: %v", err)
+			}
+			cfg := Default()
+			err = cfg.applyFileConfigWithLookup(fileCfg, emptyLookup)
+			if err == nil || !strings.Contains(err.Error(), tc.want) {
+				t.Fatalf("expected error containing %q, got %v", tc.want, err)
+			}
+		})
+	}
+}
+
+func TestApplyFileConfigOperationalSections(t *testing.T) {
+	cfg := parseAndApplyConfig(t, `
+version: 1
+concurrency:
+  total_llm: 4
+diagnostics:
+  work_dir: /tmp/notarius-test
+  retention: always
+`)
+
+	if cfg.Concurrency.TotalLLM != 4 {
+		t.Fatalf("unexpected total concurrency: %d", cfg.Concurrency.TotalLLM)
+	}
+	if cfg.Diagnostics.WorkDir != "/tmp/notarius-test" {
+		t.Fatalf("unexpected work dir: %q", cfg.Diagnostics.WorkDir)
+	}
+	if cfg.Diagnostics.Retention != diagnostics.RetentionAlways {
+		t.Fatalf("unexpected retention: %q", cfg.Diagnostics.Retention)
+	}
+}
+
+func parseAndApplyConfig(t *testing.T, raw string) Config {
+	t.Helper()
+	fileCfg, err := ParseFileConfigYAML([]byte(raw))
+	if err != nil {
+		t.Fatalf("ParseFileConfigYAML: %v", err)
+	}
+	cfg := Default()
+	if err := cfg.applyFileConfigWithLookup(fileCfg, emptyLookup); err != nil {
+		t.Fatalf("ApplyFileConfig: %v", err)
+	}
+	return cfg
+}
+
+func emptyLookup(string) (string, bool) {
+	return "", false
+}
+
+func mapLookup(values map[string]string) func(string) (string, bool) {
+	return func(key string) (string, bool) {
+		value, ok := values[key]
+		return value, ok
+	}
+}
--- a/internal/core/config/redaction.go
+++ b/internal/core/config/redaction.go
@@ -0,0 +1,14 @@
+package config
+
+const redactedSecret = "[REDACTED]"
+
+func (c Config) Redacted() Config {
+	redacted := cloneConfig(c)
+	for id, profile := range redacted.LLMProfiles {
+		if profile.APIKey != "" {
+			profile.APIKey = redactedSecret
+		}
+		redacted.LLMProfiles[id] = profile
+	}
+	return redacted
+}
--- a/internal/core/config/redaction_test.go
+++ b/internal/core/config/redaction_test.go
@@ -0,0 +1,37 @@
+package config
+
+import (
+	"testing"
+
+	"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
+)
+
+func TestRedactedConfigRemovesAPIKeyValues(t *testing.T) {
+	cfg := Default()
+	cfg.LLMProfiles[pipeline.DefaultLLMProfile] = LLMProfile{
+		Provider:       "openai-compatible",
+		BaseURL:        "https://example.invalid/v1",
+		Model:          "test-model",
+		APIKey:         "secret",
+		APIKeyEnv:      "NOTARIUS_TEST_API_KEY",
+		TimeoutSeconds: 600,
+		MaxRetries:     3,
+		MaxConcurrency: 1,
+	}
+	cfg.LLMProfiles["other"] = LLMProfile{APIKey: "other-secret", Model: "other-model"}
+
+	redacted := cfg.Redacted()
+
+	if redacted.LLMProfiles[pipeline.DefaultLLMProfile].APIKey != redactedSecret {
+		t.Fatalf("expected default API key redacted, got %+v", redacted.LLMProfiles[pipeline.DefaultLLMProfile])
+	}
+	if redacted.LLMProfiles["other"].APIKey != redactedSecret {
+		t.Fatalf("expected other API key redacted, got %+v", redacted.LLMProfiles["other"])
+	}
+	if redacted.LLMProfiles[pipeline.DefaultLLMProfile].Model != "test-model" {
+		t.Fatalf("expected non-secret fields preserved, got %+v", redacted.LLMProfiles[pipeline.DefaultLLMProfile])
+	}
+	if cfg.LLMProfiles[pipeline.DefaultLLMProfile].APIKey != "secret" {
+		t.Fatalf("redaction mutated original config")
+	}
+}