diff --git a/go.mod b/go.mod index 1d49f68..11018a1 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,5 @@ module gitea.maximumdirect.net/eric/notarius go 1.24.0 + +require gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..a62c313 --- /dev/null +++ b/go.sum @@ -0,0 +1,4 @@ +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/core/config/config.go b/internal/core/config/config.go new file mode 100644 index 0000000..e0c24fb --- /dev/null +++ b/internal/core/config/config.go @@ -0,0 +1,131 @@ +package config + +import ( + "gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" +) + +const SupportedFileConfigVersion = 1 + +type Config struct { + LLMProfiles map[string]LLMProfile `json:"llm_profiles"` + Pipelines map[string]pipeline.PipelineProfile `json:"pipelines"` + Concurrency ConcurrencyConfig `json:"concurrency"` + Diagnostics DiagnosticsConfig `json:"diagnostics"` +} + +type LLMProfile struct { + Provider string `json:"provider,omitempty"` + BaseURL string `json:"base_url,omitempty"` + Model string `json:"model,omitempty"` + APIKey string `json:"api_key,omitempty"` + APIKeyEnv string `json:"api_key_env,omitempty"` + TimeoutSeconds int `json:"timeout_seconds,omitempty"` + MaxRetries int `json:"max_retries,omitempty"` + MaxConcurrency int `json:"max_concurrency,omitempty"` +} + +type ConcurrencyConfig struct { + TotalLLM int `json:"total_llm"` +} + +type DiagnosticsConfig struct { + WorkDir string `json:"work_dir"` + Retention diagnostics.RetentionMode `json:"retention"` +} + +func Default() Config { + return Config{ + LLMProfiles: map[string]LLMProfile{ + pipeline.DefaultLLMProfile: { + Provider: "openai-compatible", + TimeoutSeconds: 600, + MaxRetries: 3, + MaxConcurrency: 1, + }, + }, + Pipelines: map[string]pipeline.PipelineProfile{}, + Concurrency: ConcurrencyConfig{ + TotalLLM: 1, + }, + Diagnostics: DiagnosticsConfig{ + WorkDir: "/tmp/notarius", + Retention: diagnostics.RetentionAuto, + }, + } +} + +func cloneConfig(in Config) Config { + out := in + out.LLMProfiles = make(map[string]LLMProfile, len(in.LLMProfiles)) + for key, profile := range in.LLMProfiles { + out.LLMProfiles[key] = profile + } + out.Pipelines = make(map[string]pipeline.PipelineProfile, len(in.Pipelines)) + for key, profile := range in.Pipelines { + out.Pipelines[key] = clonePipelineProfile(profile) + } + return out +} + +func clonePipelineProfile(in pipeline.PipelineProfile) pipeline.PipelineProfile { + out := in + out.Input = cloneModuleBinding(in.Input) + out.Chunk = cloneModuleBinding(in.Chunk) + out.Output = cloneModuleBinding(in.Output) + if len(in.Artifacts) > 0 { + out.Artifacts = make(map[string]pipeline.ArtifactLaneProfile, len(in.Artifacts)) + for key, lane := range in.Artifacts { + out.Artifacts[key] = cloneArtifactLaneProfile(lane) + } + } + return out +} + +func cloneArtifactLaneProfile(in pipeline.ArtifactLaneProfile) pipeline.ArtifactLaneProfile { + out := in + out.Extract = cloneModuleBinding(in.Extract) + out.Merge = cloneModuleBinding(in.Merge) + out.Normalize = cloneModuleBinding(in.Normalize) + if len(in.Validators) > 0 { + out.Validators = make([]pipeline.ModuleBinding, len(in.Validators)) + for i, binding := range in.Validators { + out.Validators[i] = cloneModuleBinding(binding) + } + } + return out +} + +func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding { + out := in + if len(in.Options) > 0 { + out.Options = cloneOptions(in.Options) + } + return out +} + +func cloneOptions(in map[string]any) map[string]any { + if len(in) == 0 { + return nil + } + out := make(map[string]any, len(in)) + for key, value := range in { + out[key] = cloneOptionValue(value) + } + return out +} + +func cloneOptionValue(value any) any { + switch typed := value.(type) { + case map[string]any: + return cloneOptions(typed) + case []any: + out := make([]any, len(typed)) + for i, item := range typed { + out[i] = cloneOptionValue(item) + } + return out + default: + return typed + } +} diff --git a/internal/core/config/config_test.go b/internal/core/config/config_test.go new file mode 100644 index 0000000..fffdda6 --- /dev/null +++ b/internal/core/config/config_test.go @@ -0,0 +1,78 @@ +package config + +import ( + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" +) + +func TestDefaultValues(t *testing.T) { + cfg := Default() + + defaultProfile, ok := cfg.LLMProfiles[pipeline.DefaultLLMProfile] + if !ok { + t.Fatalf("expected default LLM profile") + } + if defaultProfile.Provider != "openai-compatible" { + t.Fatalf("unexpected provider: %q", defaultProfile.Provider) + } + if defaultProfile.BaseURL != "" || defaultProfile.Model != "" { + t.Fatalf("default profile should not require base URL/model yet: %+v", defaultProfile) + } + if defaultProfile.TimeoutSeconds != 600 || defaultProfile.MaxRetries != 3 || defaultProfile.MaxConcurrency != 1 { + t.Fatalf("unexpected default LLM operational values: %+v", defaultProfile) + } + if len(cfg.Pipelines) != 0 { + t.Fatalf("expected no built-in pipeline profiles, got %v", cfg.Pipelines) + } + if cfg.Concurrency.TotalLLM != 1 { + t.Fatalf("unexpected total LLM concurrency: %d", cfg.Concurrency.TotalLLM) + } + if cfg.Diagnostics.WorkDir != "/tmp/notarius" { + t.Fatalf("unexpected diagnostics work dir: %q", cfg.Diagnostics.WorkDir) + } + if cfg.Diagnostics.Retention != diagnostics.RetentionAuto { + t.Fatalf("unexpected diagnostics retention: %q", cfg.Diagnostics.Retention) + } +} + +func TestApplyFileConfigMergesWithDefaults(t *testing.T) { + fileCfg, err := ParseFileConfigYAML([]byte(` +version: 1 +llm_profiles: + default: + model: test-model +pipelines: + example: + input: fake/input + artifacts: + events: + extract: fake/extract +`)) + if err != nil { + t.Fatalf("ParseFileConfigYAML: %v", err) + } + + cfg := Default() + if err := cfg.applyFileConfigWithLookup(fileCfg, emptyLookup); err != nil { + t.Fatalf("ApplyFileConfig: %v", err) + } + + profile := cfg.LLMProfiles[pipeline.DefaultLLMProfile] + if profile.Model != "test-model" { + t.Fatalf("expected file model, got %+v", profile) + } + if profile.Provider != "openai-compatible" || profile.TimeoutSeconds != 600 || profile.MaxRetries != 3 { + t.Fatalf("expected default LLM fields to be preserved, got %+v", profile) + } + if cfg.Concurrency.TotalLLM != 1 { + t.Fatalf("expected default concurrency preserved, got %d", cfg.Concurrency.TotalLLM) + } + if cfg.Diagnostics.Retention != diagnostics.RetentionAuto { + t.Fatalf("expected default diagnostics retention preserved, got %q", cfg.Diagnostics.Retention) + } + if _, ok := cfg.Pipelines["example"]; !ok { + t.Fatalf("expected file pipeline to be applied") + } +} diff --git a/internal/core/config/env.go b/internal/core/config/env.go new file mode 100644 index 0000000..c0de9ee --- /dev/null +++ b/internal/core/config/env.go @@ -0,0 +1,88 @@ +package config + +import ( + "fmt" + "os" + "strconv" + "strings" + + "gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" +) + +func LoadFromEnv() (Config, error) { + cfg := Default() + if err := cfg.applyEnvOverridesWithLookup(os.LookupEnv); err != nil { + return Config{}, err + } + return cfg, nil +} + +func (c *Config) ApplyEnvOverrides() error { + return c.applyEnvOverridesWithLookup(os.LookupEnv) +} + +func (c *Config) applyEnvOverridesWithLookup(lookup func(string) (string, bool)) error { + if c == nil { + return fmt.Errorf("config must not be nil") + } + if c.LLMProfiles == nil { + c.LLMProfiles = map[string]LLMProfile{} + } + + defaultProfile := c.LLMProfiles[pipeline.DefaultLLMProfile] + if raw, ok := lookup("NOTARIUS_LLM_DEFAULT_API_KEY"); ok { + defaultProfile.APIKey = raw + } + if raw, ok := lookup("NOTARIUS_LLM_DEFAULT_BASE_URL"); ok { + defaultProfile.BaseURL = strings.TrimSpace(raw) + } + if raw, ok := lookup("NOTARIUS_LLM_DEFAULT_MODEL"); ok { + defaultProfile.Model = strings.TrimSpace(raw) + } + if raw, ok := lookup("NOTARIUS_LLM_DEFAULT_TIMEOUT_SECONDS"); ok { + value, err := parseIntEnv("NOTARIUS_LLM_DEFAULT_TIMEOUT_SECONDS", raw) + if err != nil { + return err + } + defaultProfile.TimeoutSeconds = value + } + if raw, ok := lookup("NOTARIUS_LLM_DEFAULT_MAX_RETRIES"); ok { + value, err := parseIntEnv("NOTARIUS_LLM_DEFAULT_MAX_RETRIES", raw) + if err != nil { + return err + } + defaultProfile.MaxRetries = value + } + if raw, ok := lookup("NOTARIUS_LLM_DEFAULT_MAX_CONCURRENCY"); ok { + value, err := parseIntEnv("NOTARIUS_LLM_DEFAULT_MAX_CONCURRENCY", raw) + if err != nil { + return err + } + defaultProfile.MaxConcurrency = value + } + c.LLMProfiles[pipeline.DefaultLLMProfile] = defaultProfile + + if raw, ok := lookup("NOTARIUS_TOTAL_LLM_CONCURRENCY"); ok { + value, err := parseIntEnv("NOTARIUS_TOTAL_LLM_CONCURRENCY", raw) + if err != nil { + return err + } + c.Concurrency.TotalLLM = value + } + if raw, ok := lookup("NOTARIUS_WORK_DIR"); ok { + c.Diagnostics.WorkDir = strings.TrimSpace(raw) + } + if raw, ok := lookup("NOTARIUS_DIAGNOSTICS_RETENTION"); ok { + c.Diagnostics.Retention = diagnostics.RetentionMode(strings.TrimSpace(raw)) + } + return nil +} + +func parseIntEnv(name string, raw string) (int, error) { + value, err := strconv.Atoi(strings.TrimSpace(raw)) + if err != nil { + return 0, fmt.Errorf("%s: must be an integer", name) + } + return value, nil +} diff --git a/internal/core/config/env_test.go b/internal/core/config/env_test.go new file mode 100644 index 0000000..9cfb082 --- /dev/null +++ b/internal/core/config/env_test.go @@ -0,0 +1,69 @@ +package config + +import ( + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" +) + +func TestApplyEnvOverridesOperationalAndLLMValues(t *testing.T) { + cfg := Default() + cfg.Pipelines["example"] = pipeline.PipelineProfile{ID: "example", Input: pipeline.Binding("before")} + + err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{ + "NOTARIUS_LLM_DEFAULT_API_KEY": "secret", + "NOTARIUS_LLM_DEFAULT_BASE_URL": "https://example.invalid/v1", + "NOTARIUS_LLM_DEFAULT_MODEL": "test-model", + "NOTARIUS_LLM_DEFAULT_TIMEOUT_SECONDS": "120", + "NOTARIUS_LLM_DEFAULT_MAX_RETRIES": "5", + "NOTARIUS_LLM_DEFAULT_MAX_CONCURRENCY": "2", + "NOTARIUS_TOTAL_LLM_CONCURRENCY": "3", + "NOTARIUS_WORK_DIR": "/tmp/notarius-env", + "NOTARIUS_DIAGNOSTICS_RETENTION": "never", + "NOTARIUS_PIPELINE_INPUT": "after", + })) + if err != nil { + t.Fatalf("ApplyEnvOverrides: %v", err) + } + + profile := cfg.LLMProfiles[pipeline.DefaultLLMProfile] + if profile.APIKey != "secret" || profile.BaseURL != "https://example.invalid/v1" || profile.Model != "test-model" { + t.Fatalf("unexpected LLM profile strings: %+v", profile) + } + if profile.TimeoutSeconds != 120 || profile.MaxRetries != 5 || profile.MaxConcurrency != 2 { + t.Fatalf("unexpected LLM profile numeric values: %+v", profile) + } + if cfg.Concurrency.TotalLLM != 3 { + t.Fatalf("unexpected total concurrency: %d", cfg.Concurrency.TotalLLM) + } + if cfg.Diagnostics.WorkDir != "/tmp/notarius-env" || cfg.Diagnostics.Retention != diagnostics.RetentionNever { + t.Fatalf("unexpected diagnostics config: %+v", cfg.Diagnostics) + } + if cfg.Pipelines["example"].Input.Module != "before" { + t.Fatalf("environment overrides must not change pipeline wiring: %+v", cfg.Pipelines["example"]) + } +} + +func TestApplyEnvOverridesRejectsInvalidIntegers(t *testing.T) { + cfg := Default() + err := cfg.applyEnvOverridesWithLookup(mapLookup(map[string]string{ + "NOTARIUS_TOTAL_LLM_CONCURRENCY": "many", + })) + if err == nil || !strings.Contains(err.Error(), "NOTARIUS_TOTAL_LLM_CONCURRENCY") { + t.Fatalf("expected named integer error, got %v", err) + } +} + +func TestLoadFromEnvUsesDefaultConfig(t *testing.T) { + t.Setenv("NOTARIUS_LLM_DEFAULT_MODEL", "env-model") + + cfg, err := LoadFromEnv() + if err != nil { + t.Fatalf("LoadFromEnv: %v", err) + } + if cfg.LLMProfiles[pipeline.DefaultLLMProfile].Model != "env-model" { + t.Fatalf("expected env model, got %+v", cfg.LLMProfiles[pipeline.DefaultLLMProfile]) + } +} diff --git a/internal/core/config/file_config.go b/internal/core/config/file_config.go new file mode 100644 index 0000000..469031d --- /dev/null +++ b/internal/core/config/file_config.go @@ -0,0 +1,308 @@ +package config + +import ( + "bytes" + "fmt" + "os" + "regexp" + "strings" + "time" + + "gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics" + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" + "gopkg.in/yaml.v3" +) + +var envVarNamePattern = regexp.MustCompile(`^[A-Za-z_][A-Za-z0-9_]*$`) + +type FileConfig struct { + Version int `yaml:"version"` + LLMProfiles map[string]FileLLMProfile `yaml:"llm_profiles,omitempty"` + Pipelines map[string]FilePipelineProfile `yaml:"pipelines,omitempty"` + Concurrency *FileConcurrencyConfig `yaml:"concurrency,omitempty"` + Diagnostics *FileDiagnosticsConfig `yaml:"diagnostics,omitempty"` +} + +type FileLLMProfile struct { + Provider *string `yaml:"provider,omitempty"` + BaseURL *string `yaml:"base_url,omitempty"` + Model *string `yaml:"model,omitempty"` + APIKeyEnv *string `yaml:"api_key_env,omitempty"` + Timeout *fileDurationSeconds `yaml:"timeout,omitempty"` + MaxRetries *int `yaml:"max_retries,omitempty"` + MaxConcurrency *int `yaml:"max_concurrency,omitempty"` +} + +type FilePipelineProfile struct { + Input fileModuleBinding `yaml:"input"` + Chunk *fileModuleBinding `yaml:"chunk,omitempty"` + Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"` + Output *fileModuleBinding `yaml:"output,omitempty"` +} + +type FileArtifactLaneProfile struct { + Extract fileModuleBinding `yaml:"extract"` + Merge *fileModuleBinding `yaml:"merge,omitempty"` + Normalize *fileModuleBinding `yaml:"normalize,omitempty"` + Validators []fileModuleBinding `yaml:"validators,omitempty"` +} + +type FileConcurrencyConfig struct { + TotalLLM *int `yaml:"total_llm,omitempty"` +} + +type FileDiagnosticsConfig struct { + WorkDir *string `yaml:"work_dir,omitempty"` + Retention *string `yaml:"retention,omitempty"` +} + +type fileDurationSeconds struct { + seconds int +} + +func (d *fileDurationSeconds) UnmarshalYAML(node *yaml.Node) error { + if node.Kind != yaml.ScalarNode { + return fmt.Errorf("must be an integer seconds value or duration string") + } + if node.Tag == "!!int" { + var seconds int + if err := node.Decode(&seconds); err != nil { + return fmt.Errorf("must be an integer seconds value or duration string") + } + d.seconds = seconds + return nil + } + + var raw string + if err := node.Decode(&raw); err != nil { + return fmt.Errorf("must be an integer seconds value or duration string") + } + duration, err := time.ParseDuration(strings.TrimSpace(raw)) + if err != nil { + return fmt.Errorf("invalid duration %q", raw) + } + if duration%time.Second != 0 { + return fmt.Errorf("duration %q must resolve to whole seconds", raw) + } + d.seconds = int(duration / time.Second) + return nil +} + +func (d fileDurationSeconds) Seconds() int { + return d.seconds +} + +type fileModuleBinding struct { + Module string + LLMProfile string + Options map[string]any +} + +func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error { + switch node.Kind { + case yaml.ScalarNode: + var module string + if err := node.Decode(&module); err != nil { + return fmt.Errorf("module binding must be a string or object") + } + b.Module = strings.TrimSpace(module) + return nil + case yaml.MappingNode: + for i := 0; i < len(node.Content); i += 2 { + keyNode := node.Content[i] + valueNode := node.Content[i+1] + switch keyNode.Value { + case "module": + var module string + if err := valueNode.Decode(&module); err != nil { + return err + } + b.Module = strings.TrimSpace(module) + case "llm_profile": + var llmProfile string + if err := valueNode.Decode(&llmProfile); err != nil { + return err + } + b.LLMProfile = strings.TrimSpace(llmProfile) + case "options": + var options map[string]any + if err := valueNode.Decode(&options); err != nil { + return err + } + b.Options = normalizeOptions(options) + default: + return fmt.Errorf("field %s not found in module binding", keyNode.Value) + } + } + return nil + default: + return fmt.Errorf("module binding must be a string or object") + } +} + +func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding { + return pipeline.ModuleBinding{ + Module: strings.TrimSpace(b.Module), + LLMProfile: strings.TrimSpace(b.LLMProfile), + Options: cloneOptions(b.Options), + } +} + +func LoadFileConfig(path string) (FileConfig, error) { + data, err := os.ReadFile(path) + if err != nil { + return FileConfig{}, fmt.Errorf("read config file %q: %w", path, err) + } + cfg, err := ParseFileConfigYAML(data) + if err != nil { + return FileConfig{}, fmt.Errorf("parse config file %q: %w", path, err) + } + return cfg, nil +} + +func ParseFileConfigYAML(data []byte) (FileConfig, error) { + var fileCfg FileConfig + decoder := yaml.NewDecoder(bytes.NewReader(data)) + decoder.KnownFields(true) + if err := decoder.Decode(&fileCfg); err != nil { + return FileConfig{}, fmt.Errorf("decode yaml: %w", err) + } + if fileCfg.Version == 0 { + return FileConfig{}, fmt.Errorf("config version is required") + } + if fileCfg.Version != SupportedFileConfigVersion { + return FileConfig{}, fmt.Errorf("unsupported config version %d", fileCfg.Version) + } + return fileCfg, nil +} + +func (c *Config) ApplyFileConfig(fileCfg FileConfig) error { + return c.applyFileConfigWithLookup(fileCfg, os.LookupEnv) +} + +func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(string) (string, bool)) error { + if c == nil { + return fmt.Errorf("config must not be nil") + } + if fileCfg.Version != SupportedFileConfigVersion { + return fmt.Errorf("unsupported config version %d", fileCfg.Version) + } + if c.LLMProfiles == nil { + c.LLMProfiles = map[string]LLMProfile{} + } + if c.Pipelines == nil { + c.Pipelines = map[string]pipeline.PipelineProfile{} + } + + for rawID, fileProfile := range fileCfg.LLMProfiles { + profileID := strings.TrimSpace(rawID) + if profileID == "" { + return fmt.Errorf("llm profile id must not be empty") + } + profile := c.LLMProfiles[profileID] + if fileProfile.Provider != nil { + profile.Provider = strings.TrimSpace(*fileProfile.Provider) + } + if fileProfile.BaseURL != nil { + profile.BaseURL = strings.TrimSpace(*fileProfile.BaseURL) + } + if fileProfile.Model != nil { + profile.Model = strings.TrimSpace(*fileProfile.Model) + } + if fileProfile.APIKeyEnv != nil { + apiKey, err := resolveAPIKeyEnv(*fileProfile.APIKeyEnv, lookup) + if err != nil { + return fmt.Errorf("llm_profiles.%s.api_key_env: %w", profileID, err) + } + profile.APIKeyEnv = strings.TrimSpace(*fileProfile.APIKeyEnv) + profile.APIKey = apiKey + } + if fileProfile.Timeout != nil { + profile.TimeoutSeconds = fileProfile.Timeout.Seconds() + } + if fileProfile.MaxRetries != nil { + profile.MaxRetries = *fileProfile.MaxRetries + } + if fileProfile.MaxConcurrency != nil { + profile.MaxConcurrency = *fileProfile.MaxConcurrency + } + c.LLMProfiles[profileID] = profile + } + + for rawID, filePipeline := range fileCfg.Pipelines { + pipelineID := strings.TrimSpace(rawID) + if pipelineID == "" { + return fmt.Errorf("pipeline id must not be empty") + } + profile := pipeline.PipelineProfile{ + ID: pipelineID, + Input: filePipeline.Input.toPipelineBinding(), + Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)), + } + if filePipeline.Chunk != nil { + profile.Chunk = filePipeline.Chunk.toPipelineBinding() + } + if filePipeline.Output != nil { + profile.Output = filePipeline.Output.toPipelineBinding() + } + for rawLaneID, fileLane := range filePipeline.Artifacts { + laneID := strings.TrimSpace(rawLaneID) + if laneID == "" { + return fmt.Errorf("pipeline %q artifact lane id must not be empty", pipelineID) + } + lane := pipeline.ArtifactLaneProfile{ + Extract: fileLane.Extract.toPipelineBinding(), + } + if fileLane.Merge != nil { + lane.Merge = fileLane.Merge.toPipelineBinding() + } + if fileLane.Normalize != nil { + lane.Normalize = fileLane.Normalize.toPipelineBinding() + } + if len(fileLane.Validators) > 0 { + lane.Validators = make([]pipeline.ModuleBinding, len(fileLane.Validators)) + for i, validator := range fileLane.Validators { + lane.Validators[i] = validator.toPipelineBinding() + } + } + profile.Artifacts[laneID] = lane + } + c.Pipelines[pipelineID] = profile + } + + if fileCfg.Concurrency != nil && fileCfg.Concurrency.TotalLLM != nil { + c.Concurrency.TotalLLM = *fileCfg.Concurrency.TotalLLM + } + if fileCfg.Diagnostics != nil { + if fileCfg.Diagnostics.WorkDir != nil { + c.Diagnostics.WorkDir = strings.TrimSpace(*fileCfg.Diagnostics.WorkDir) + } + if fileCfg.Diagnostics.Retention != nil { + c.Diagnostics.Retention = diagnostics.RetentionMode(strings.TrimSpace(*fileCfg.Diagnostics.Retention)) + } + } + + return nil +} + +func resolveAPIKeyEnv(envName string, lookup func(string) (string, bool)) (string, error) { + name := strings.TrimSpace(envName) + if name == "" { + return "", fmt.Errorf("must not be empty") + } + if !envVarNamePattern.MatchString(name) { + return "", fmt.Errorf("must be an environment variable name") + } + value, ok := lookup(name) + if !ok { + return "", fmt.Errorf("%s is not set", name) + } + return value, nil +} + +func normalizeOptions(options map[string]any) map[string]any { + if len(options) == 0 { + return nil + } + return cloneOptions(options) +} diff --git a/internal/core/config/file_config_test.go b/internal/core/config/file_config_test.go new file mode 100644 index 0000000..92e0b27 --- /dev/null +++ b/internal/core/config/file_config_test.go @@ -0,0 +1,310 @@ +package config + +import ( + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics" +) + +func TestParseMinimalValidConfig(t *testing.T) { + fileCfg, err := ParseFileConfigYAML([]byte(` +version: 1 +`)) + if err != nil { + t.Fatalf("ParseFileConfigYAML: %v", err) + } + if fileCfg.Version != SupportedFileConfigVersion { + t.Fatalf("unexpected version: %d", fileCfg.Version) + } +} + +func TestLoadFileConfig(t *testing.T) { + path := filepath.Join(t.TempDir(), "config.yml") + if err := os.WriteFile(path, []byte("version: 1\n"), 0o644); err != nil { + t.Fatalf("write config: %v", err) + } + + fileCfg, err := LoadFileConfig(path) + if err != nil { + t.Fatalf("LoadFileConfig: %v", err) + } + if fileCfg.Version != SupportedFileConfigVersion { + t.Fatalf("unexpected version: %d", fileCfg.Version) + } +} + +func TestParseFileConfigRejectsUnknownYAMLFields(t *testing.T) { + _, err := ParseFileConfigYAML([]byte(` +version: 1 +unexpected: true +`)) + if err == nil || !strings.Contains(err.Error(), "field unexpected not found") { + t.Fatalf("expected unknown field error, got %v", err) + } +} + +func TestParseFileConfigRejectsUnknownModuleBindingFields(t *testing.T) { + _, err := ParseFileConfigYAML([]byte(` +version: 1 +pipelines: + example: + input: + module: fake/input + unexpected: true + artifacts: + events: + extract: fake/extract +`)) + if err == nil || !strings.Contains(err.Error(), "field unexpected not found") { + t.Fatalf("expected unknown binding field error, got %v", err) + } +} + +func TestParseFileConfigRejectsMissingAndUnsupportedVersion(t *testing.T) { + tests := []struct { + name string + data string + want string + }{ + {name: "missing", data: `llm_profiles: {}`, want: "version is required"}, + {name: "unsupported", data: `version: 2`, want: "unsupported config version"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + _, err := ParseFileConfigYAML([]byte(tc.data)) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("expected error containing %q, got %v", tc.want, err) + } + }) + } +} + +func TestParseFileConfigModuleBindingForms(t *testing.T) { + cfg := parseAndApplyConfig(t, ` +version: 1 +pipelines: + example: + input: fake/input + chunk: + module: generic + options: + size: 10 + flags: + - alpha + nested: + enabled: true + artifacts: + events: + extract: + module: fake/extract + llm_profile: fast + options: + temperature: 0 + merge: appendorder + normalize: + module: noop + output: json +`) + + profile := cfg.Pipelines["example"] + if profile.Input.Module != "fake/input" { + t.Fatalf("unexpected input binding: %+v", profile.Input) + } + if profile.Chunk.Module != "generic" { + t.Fatalf("unexpected chunk binding: %+v", profile.Chunk) + } + if profile.Chunk.Options["size"] != 10 { + t.Fatalf("expected chunk options to preserve scalar, got %#v", profile.Chunk.Options) + } + if !reflect.DeepEqual(profile.Chunk.Options["flags"], []any{"alpha"}) { + t.Fatalf("expected list option, got %#v", profile.Chunk.Options["flags"]) + } + nested, ok := profile.Chunk.Options["nested"].(map[string]any) + if !ok || nested["enabled"] != true { + t.Fatalf("expected nested map option, got %#v", profile.Chunk.Options["nested"]) + } + + lane := profile.Artifacts["events"] + if lane.Extract.Module != "fake/extract" || lane.Extract.LLMProfile != "fast" { + t.Fatalf("unexpected extract binding: %+v", lane.Extract) + } + if lane.Extract.Options["temperature"] != 0 { + t.Fatalf("expected object options, got %#v", lane.Extract.Options) + } + if lane.Merge.Module != "appendorder" || lane.Normalize.Module != "noop" { + t.Fatalf("unexpected lane defaults: %+v", lane) + } + if profile.Output.Module != "json" { + t.Fatalf("unexpected output binding: %+v", profile.Output) + } +} + +func TestParseFileConfigValidatorMixedBindingForms(t *testing.T) { + cfg := parseAndApplyConfig(t, ` +version: 1 +pipelines: + example: + input: fake/input + artifacts: + events: + extract: fake/extract + validators: + - fake/validator + - module: fake/llm-validator + llm_profile: careful + options: + threshold: 0.7 +`) + + validators := cfg.Pipelines["example"].Artifacts["events"].Validators + if len(validators) != 2 { + t.Fatalf("expected two validators, got %d", len(validators)) + } + if validators[0].Module != "fake/validator" { + t.Fatalf("unexpected shorthand validator: %+v", validators[0]) + } + if validators[1].Module != "fake/llm-validator" || validators[1].LLMProfile != "careful" { + t.Fatalf("unexpected object validator: %+v", validators[1]) + } + if validators[1].Options["threshold"] != 0.7 { + t.Fatalf("unexpected validator options: %#v", validators[1].Options) + } +} + +func TestParseFileConfigDurationParsing(t *testing.T) { + tests := []struct { + name string + raw string + want int + }{ + {name: "integer seconds", raw: "600", want: 600}, + {name: "duration string", raw: "10m", want: 600}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cfg := parseAndApplyConfig(t, ` +version: 1 +llm_profiles: + default: + timeout: `+tc.raw+` +`) + if got := cfg.LLMProfiles["default"].TimeoutSeconds; got != tc.want { + t.Fatalf("TimeoutSeconds = %d, want %d", got, tc.want) + } + }) + } +} + +func TestParseFileConfigRejectsSubsecondDuration(t *testing.T) { + _, err := ParseFileConfigYAML([]byte(` +version: 1 +llm_profiles: + default: + timeout: 1500ms +`)) + if err == nil || !strings.Contains(err.Error(), "whole seconds") { + t.Fatalf("expected whole-seconds duration error, got %v", err) + } +} + +func TestApplyFileConfigResolvesAPIKeyEnv(t *testing.T) { + fileCfg, err := ParseFileConfigYAML([]byte(` +version: 1 +llm_profiles: + default: + api_key_env: NOTARIUS_TEST_API_KEY +`)) + if err != nil { + t.Fatalf("ParseFileConfigYAML: %v", err) + } + + cfg := Default() + if err := cfg.applyFileConfigWithLookup(fileCfg, mapLookup(map[string]string{"NOTARIUS_TEST_API_KEY": "secret"})); err != nil { + t.Fatalf("ApplyFileConfig: %v", err) + } + profile := cfg.LLMProfiles["default"] + if profile.APIKeyEnv != "NOTARIUS_TEST_API_KEY" || profile.APIKey != "secret" { + t.Fatalf("unexpected resolved API key: %+v", profile) + } +} + +func TestApplyFileConfigRejectsInvalidAPIKeyEnv(t *testing.T) { + tests := []struct { + name string + env string + want string + }{ + {name: "invalid name", env: "NOTARIUS-KEY", want: "environment variable name"}, + {name: "not set", env: "NOTARIUS_TEST_API_KEY", want: "is not set"}, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + fileCfg, err := ParseFileConfigYAML([]byte(` +version: 1 +llm_profiles: + default: + api_key_env: ` + tc.env + ` +`)) + if err != nil { + t.Fatalf("ParseFileConfigYAML: %v", err) + } + cfg := Default() + err = cfg.applyFileConfigWithLookup(fileCfg, emptyLookup) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("expected error containing %q, got %v", tc.want, err) + } + }) + } +} + +func TestApplyFileConfigOperationalSections(t *testing.T) { + cfg := parseAndApplyConfig(t, ` +version: 1 +concurrency: + total_llm: 4 +diagnostics: + work_dir: /tmp/notarius-test + retention: always +`) + + if cfg.Concurrency.TotalLLM != 4 { + t.Fatalf("unexpected total concurrency: %d", cfg.Concurrency.TotalLLM) + } + if cfg.Diagnostics.WorkDir != "/tmp/notarius-test" { + t.Fatalf("unexpected work dir: %q", cfg.Diagnostics.WorkDir) + } + if cfg.Diagnostics.Retention != diagnostics.RetentionAlways { + t.Fatalf("unexpected retention: %q", cfg.Diagnostics.Retention) + } +} + +func parseAndApplyConfig(t *testing.T, raw string) Config { + t.Helper() + fileCfg, err := ParseFileConfigYAML([]byte(raw)) + if err != nil { + t.Fatalf("ParseFileConfigYAML: %v", err) + } + cfg := Default() + if err := cfg.applyFileConfigWithLookup(fileCfg, emptyLookup); err != nil { + t.Fatalf("ApplyFileConfig: %v", err) + } + return cfg +} + +func emptyLookup(string) (string, bool) { + return "", false +} + +func mapLookup(values map[string]string) func(string) (string, bool) { + return func(key string) (string, bool) { + value, ok := values[key] + return value, ok + } +} diff --git a/internal/core/config/redaction.go b/internal/core/config/redaction.go new file mode 100644 index 0000000..ea220dc --- /dev/null +++ b/internal/core/config/redaction.go @@ -0,0 +1,14 @@ +package config + +const redactedSecret = "[REDACTED]" + +func (c Config) Redacted() Config { + redacted := cloneConfig(c) + for id, profile := range redacted.LLMProfiles { + if profile.APIKey != "" { + profile.APIKey = redactedSecret + } + redacted.LLMProfiles[id] = profile + } + return redacted +} diff --git a/internal/core/config/redaction_test.go b/internal/core/config/redaction_test.go new file mode 100644 index 0000000..5e39793 --- /dev/null +++ b/internal/core/config/redaction_test.go @@ -0,0 +1,37 @@ +package config + +import ( + "testing" + + "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" +) + +func TestRedactedConfigRemovesAPIKeyValues(t *testing.T) { + cfg := Default() + cfg.LLMProfiles[pipeline.DefaultLLMProfile] = LLMProfile{ + Provider: "openai-compatible", + BaseURL: "https://example.invalid/v1", + Model: "test-model", + APIKey: "secret", + APIKeyEnv: "NOTARIUS_TEST_API_KEY", + TimeoutSeconds: 600, + MaxRetries: 3, + MaxConcurrency: 1, + } + cfg.LLMProfiles["other"] = LLMProfile{APIKey: "other-secret", Model: "other-model"} + + redacted := cfg.Redacted() + + if redacted.LLMProfiles[pipeline.DefaultLLMProfile].APIKey != redactedSecret { + t.Fatalf("expected default API key redacted, got %+v", redacted.LLMProfiles[pipeline.DefaultLLMProfile]) + } + if redacted.LLMProfiles["other"].APIKey != redactedSecret { + t.Fatalf("expected other API key redacted, got %+v", redacted.LLMProfiles["other"]) + } + if redacted.LLMProfiles[pipeline.DefaultLLMProfile].Model != "test-model" { + t.Fatalf("expected non-secret fields preserved, got %+v", redacted.LLMProfiles[pipeline.DefaultLLMProfile]) + } + if cfg.LLMProfiles[pipeline.DefaultLLMProfile].APIKey != "secret" { + t.Fatalf("redaction mutated original config") + } +}