package config import ( "fmt" "strings" "testing" "gitea.maximumdirect.net/eric/notarius/internal/core/diagnostics" "gitea.maximumdirect.net/eric/notarius/internal/framework/contracts" "gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline" ) func TestValidateSuccessForValidConfig(t *testing.T) { cfg := validConfig() if err := cfg.Validate(); err != nil { t.Fatalf("Validate: %v", err) } } func TestValidateAllowsExplicitScriptoriumProfileIDOnBinding(t *testing.T) { cfg := validConfig() lane := cfg.Pipelines["example"].Artifacts["events"] lane.Extract.LLMProfile = "scriptorium-profile" cfg.Pipelines["example"].Artifacts["events"] = lane if err := cfg.Validate(); err != nil { t.Fatalf("Validate() error = %v, want nil", err) } } func TestValidateRejectsWhitespaceOnlyExplicitLLMProfile(t *testing.T) { cfg := validConfig() lane := cfg.Pipelines["example"].Artifacts["events"] lane.Extract.LLMProfile = " " cfg.Pipelines["example"].Artifacts["events"] = lane err := cfg.Validate() if err == nil || !strings.Contains(err.Error(), "llm_profile") || !strings.Contains(err.Error(), "events") { t.Fatalf("expected llm_profile error with lane context, got %v", err) } } func TestValidateRejectsInvalidNumericFields(t *testing.T) { tests := []struct { name string mutate func(Config) Config want string }{ { name: "total concurrency", mutate: func(cfg Config) Config { cfg.Concurrency.TotalLLM = 0 return cfg }, want: "total LLM concurrency", }, { name: "negative retries", mutate: func(cfg Config) Config { profile := cfg.Pipelines["example"] lane := profile.Artifacts["events"] lane.Merge.Retries = -1 profile.Artifacts["events"] = lane cfg.Pipelines["example"] = profile return cfg }, want: "retries", }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { err := tc.mutate(validConfig()).Validate() if err == nil || !strings.Contains(err.Error(), tc.want) { t.Fatalf("expected error containing %q, got %v", tc.want, err) } }) } } func TestValidateStageWorkerBoundaries(t *testing.T) { for _, test := range []struct { name string workers int wantErr bool }{ {name: "below minimum", workers: 0, wantErr: true}, {name: "minimum", workers: 1}, {name: "maximum", workers: 4}, {name: "above maximum", workers: 5, wantErr: true}, } { t.Run(test.name, func(t *testing.T) { cfg := validConfig() cfg.Concurrency.TotalLLM = 4 cfg.Concurrency.StageWorkers["extract"] = test.workers err := cfg.Validate() if test.wantErr && (err == nil || !strings.Contains(err.Error(), "stage_workers.extract")) { t.Fatalf("Validate() error = %v, want extract worker range error", err) } if !test.wantErr && err != nil { t.Fatalf("Validate() error = %v, want nil", err) } }) } } func TestValidateRejectsUnknownEffectiveStageWorkerKey(t *testing.T) { cfg := validConfig() cfg.Concurrency.StageWorkers["merge"] = 1 err := cfg.Validate() if err == nil || !strings.Contains(err.Error(), "stage_workers key") || !strings.Contains(err.Error(), "merge") { t.Fatalf("Validate() error = %v, want unknown stage worker key", err) } } func TestValidateRejectsMutuallyExclusiveScriptoriumProfileSources(t *testing.T) { cfg := validConfig() cfg.Scriptorium.ProfileDir = "./profiles" cfg.Scriptorium.ProfileFile = "./profiles.yml" err := cfg.Validate() if err == nil || !strings.Contains(err.Error(), "mutually exclusive") { t.Fatalf("expected Scriptorium source conflict, got %v", err) } } func TestValidateRejectsInvalidDiagnosticsRetention(t *testing.T) { cfg := validConfig() cfg.Diagnostics.Retention = diagnostics.RetentionMode("sometimes") err := cfg.Validate() if err == nil || !strings.Contains(err.Error(), "retention") { t.Fatalf("expected retention error, got %v", err) } } func TestValidateRejectsInvalidWorkspaceDiagnosticsRetention(t *testing.T) { cfg := validConfig() cfg.Workspace.Diagnostics.Retention = diagnostics.RetentionMode("sometimes") cfg.Workspace.Diagnostics.retentionSet = true err := cfg.Validate() if err == nil || !strings.Contains(err.Error(), "workspace diagnostics retention") { t.Fatalf("expected workspace retention error, got %v", err) } } func TestValidateRejectsInvalidReferenceMaps(t *testing.T) { tests := []struct { name string mutate func(Config) Config want []string }{ { name: "empty chunk slot", mutate: func(cfg Config) Config { profile := cfg.Pipelines["example"] profile.Chunk.References = map[string]string{" ": "./roster.yml"} cfg.Pipelines["example"] = profile return cfg }, want: []string{"example", "chunk", "reference slot", "empty"}, }, { name: "empty chunk source", mutate: func(cfg Config) Config { profile := cfg.Pipelines["example"] profile.Chunk.References = map[string]string{"roster": " "} cfg.Pipelines["example"] = profile return cfg }, want: []string{"example", "chunk", "roster", "source", "empty"}, }, { name: "empty extract slot", mutate: func(cfg Config) Config { profile := cfg.Pipelines["example"] lane := profile.Artifacts["events"] lane.Extract.References = map[string]string{" ": "./roster.yml"} profile.Artifacts["events"] = lane cfg.Pipelines["example"] = profile return cfg }, want: []string{"example", "events", "extract", "reference slot", "empty"}, }, { name: "empty extract source", mutate: func(cfg Config) Config { profile := cfg.Pipelines["example"] lane := profile.Artifacts["events"] lane.Extract.References = map[string]string{"roster": " "} profile.Artifacts["events"] = lane cfg.Pipelines["example"] = profile return cfg }, want: []string{"example", "events", "extract", "roster", "source", "empty"}, }, { name: "empty normalize slot", mutate: func(cfg Config) Config { profile := cfg.Pipelines["example"] lane := profile.Artifacts["events"] lane.Normalize.References = map[string]string{" ": "./roster.yml"} profile.Artifacts["events"] = lane cfg.Pipelines["example"] = profile return cfg }, want: []string{"example", "events", "normalize", "reference slot", "empty"}, }, { name: "empty normalize source", mutate: func(cfg Config) Config { profile := cfg.Pipelines["example"] lane := profile.Artifacts["events"] lane.Normalize.References = map[string]string{"roster": " "} profile.Artifacts["events"] = lane cfg.Pipelines["example"] = profile return cfg }, want: []string{"example", "events", "normalize", "roster", "source", "empty"}, }, { name: "empty pipeline slot", mutate: func(cfg Config) Config { profile := cfg.Pipelines["example"] profile.References = map[string]string{" ": "./roster.yml"} cfg.Pipelines["example"] = profile return cfg }, want: []string{"example", "reference slot", "empty"}, }, { name: "empty pipeline source", mutate: func(cfg Config) Config { profile := cfg.Pipelines["example"] profile.References = map[string]string{"roster": " "} cfg.Pipelines["example"] = profile return cfg }, want: []string{"example", "roster", "source", "empty"}, }, { name: "empty lane slot", mutate: func(cfg Config) Config { profile := cfg.Pipelines["example"] lane := profile.Artifacts["events"] lane.References = map[string]string{" ": "./roster.yml"} profile.Artifacts["events"] = lane cfg.Pipelines["example"] = profile return cfg }, want: []string{"example", "events", "reference slot", "empty"}, }, { name: "empty lane source", mutate: func(cfg Config) Config { profile := cfg.Pipelines["example"] lane := profile.Artifacts["events"] lane.References = map[string]string{"roster": " "} profile.Artifacts["events"] = lane cfg.Pipelines["example"] = profile return cfg }, want: []string{"example", "events", "roster", "source", "empty"}, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { err := tc.mutate(validConfig()).Validate() if err == nil { t.Fatal("Validate() error = nil, want error") } for _, want := range tc.want { if !strings.Contains(err.Error(), want) { t.Fatalf("Validate() error = %q, want substring %q", err.Error(), want) } } }) } } func TestValidateRejectsReferencesOnUnsupportedBindings(t *testing.T) { tests := []struct { name string mutate func(Config) Config want []string }{ { name: "input", mutate: func(cfg Config) Config { profile := cfg.Pipelines["example"] profile.Input.References = map[string]string{"roster": "./roster.yml"} cfg.Pipelines["example"] = profile return cfg }, want: []string{"example", "input", "references", "not supported"}, }, { name: "output", mutate: func(cfg Config) Config { profile := cfg.Pipelines["example"] profile.Output.References = map[string]string{"roster": "./roster.yml"} cfg.Pipelines["example"] = profile return cfg }, want: []string{"example", "output", "references", "not supported"}, }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { err := tc.mutate(validConfig()).Validate() if err == nil { t.Fatal("Validate() error = nil, want error") } for _, want := range tc.want { if !strings.Contains(err.Error(), want) { t.Fatalf("Validate() error = %q, want substring %q", err.Error(), want) } } }) } } func TestValidateRejectsEmptyIDs(t *testing.T) { tests := []struct { name string mutate func(Config) Config want string }{ { name: "pipeline", mutate: func(cfg Config) Config { cfg.Pipelines[" "] = pipeline.PipelineProfile{} return cfg }, want: "pipeline id", }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { err := tc.mutate(validConfig()).Validate() if err == nil || !strings.Contains(err.Error(), tc.want) { t.Fatalf("expected error containing %q, got %v", tc.want, err) } }) } } func TestValidateRejectsIDsDuplicatedAfterTrimming(t *testing.T) { tests := []struct { name string mutate func(Config) Config want string }{ { name: "pipeline", mutate: func(cfg Config) Config { cfg.Pipelines[" example "] = cfg.Pipelines["example"] return cfg }, want: "duplicated", }, } for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { err := tc.mutate(validConfig()).Validate() if err == nil || !strings.Contains(err.Error(), tc.want) { t.Fatalf("expected error containing %q, got %v", tc.want, err) } }) } } func TestValidateRejectsConfiguredValidators(t *testing.T) { cfg := validConfig() profile := cfg.Pipelines["example"] lane := profile.Artifacts["events"] lane.Validators = []pipeline.ModuleBinding{pipeline.Binding("fake/validator")} profile.Artifacts["events"] = lane cfg.Pipelines["example"] = profile err := cfg.Validate() if err == nil { t.Fatal("Validate() error = nil, want configured validators error") } for _, want := range []string{"example", "events", "validators", "extract.validators", "merge.validators", "normalize.validators"} { if !strings.Contains(err.Error(), want) { t.Fatalf("Validate() error = %q, want substring %q", err.Error(), want) } } } func TestValidateAcceptsStageLocalValidatorOverrides(t *testing.T) { cfg := validConfig() profile := cfg.Pipelines["example"] profile.Chunk.Validators = pipeline.ValidatorOverride{Set: true} lane := profile.Artifacts["events"] lane.Extract.Validators = pipeline.ValidatorOverride{ Set: true, Validators: []pipeline.ModuleBinding{ pipeline.Binding("fake/validator"), {Module: "fake/llm-validator", LLMProfile: "careful", Options: map[string]any{"threshold": 0.7}}, }, } lane.Merge.Validators = pipeline.ValidatorOverride{Set: true} profile.Artifacts["events"] = lane cfg.Pipelines["example"] = profile if err := cfg.Validate(); err != nil { t.Fatalf("Validate() error = %v, want nil", err) } } func TestValidateRejectsInvalidValidatorBindings(t *testing.T) { tests := []struct { name string binding pipeline.ModuleBinding want string }{ { name: "empty module", binding: pipeline.ModuleBinding{}, want: "module must not be empty", }, { name: "references", binding: pipeline.ModuleBinding{Module: "fake/validator", References: map[string]string{"roster": "./roster.txt"}}, want: "references are not supported", }, { name: "nested validators", binding: pipeline.ModuleBinding{Module: "fake/validator", Validators: pipeline.ValidatorOverride{Set: true}}, want: "nested validators are not supported", }, { name: "retries", binding: pipeline.ModuleBinding{Module: "fake/validator", Retries: 1}, want: "retries are not supported", }, } for _, test := range tests { t.Run(test.name, func(t *testing.T) { cfg := validConfig() profile := cfg.Pipelines["example"] lane := profile.Artifacts["events"] lane.Extract.Validators = pipeline.ValidatorOverride{ Set: true, Validators: []pipeline.ModuleBinding{test.binding}, } profile.Artifacts["events"] = lane cfg.Pipelines["example"] = profile err := cfg.Validate() if err == nil || !strings.Contains(err.Error(), test.want) { t.Fatalf("Validate() error = %v, want %q", err, test.want) } }) } } func validConfig() Config { cfg := Default() cfg.Pipelines["example"] = pipeline.PipelineProfile{ Input: pipeline.Binding("fake/input"), Artifacts: map[string]pipeline.ArtifactLaneProfile{ "events": { Extract: pipeline.Binding("fake/extract"), }, "notes": { Extract: pipeline.Binding("fake/extract"), }, }, } return cfg } func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.ModuleCatalog { t.Helper() specs := map[string]pipeline.ModuleSpec{ "fake/input": { Key: "fake/input", Stage: pipeline.StageInput, Provides: []string{"source"}, }, "generic": { Key: "generic", Stage: pipeline.StageChunk, Requires: []string{"source"}, Provides: []string{"chunks"}, }, "fake/extract": { Key: "fake/extract", Stage: pipeline.StageExtract, Requires: []string{"chunks"}, Provides: []string{"artifact"}, }, "appendorder": { Key: "appendorder", Stage: pipeline.StageMerge, Requires: []string{"artifact"}, Provides: []string{"merged"}, }, "noop": { Key: "noop", Stage: pipeline.StageNormalize, Requires: []string{"merged"}, Provides: []string{"normalized"}, }, "fake/validator": { Key: "fake/validator", Stage: pipeline.StageValidate, Requires: []string{"normalized"}, Provides: []string{"validated"}, }, "fake/llm-validator": { Key: "fake/llm-validator", Stage: pipeline.StageValidate, Requires: []string{"normalized"}, Provides: []string{"validated"}, }, "json": { Key: "json", Stage: pipeline.StageOutput, Requires: []string{"normalized"}, }, } for _, override := range overrides { specs[override.Key] = override } for _, key := range []string{"fake/extract", "appendorder", "noop"} { spec := specs[key] spec.ArtifactKind = fakeArtifactKind specs[key] = spec } inputs := pipeline.NewInputAdapterRegistry() chunkers := pipeline.NewChunkerRegistry() extractors := pipeline.NewExtractorRegistry() mergers := pipeline.NewMergerRegistry() normalizers := pipeline.NewNormalizerRegistry() validators := pipeline.NewValidatorRegistry() outputs := pipeline.NewOutputEncoderRegistry() mustRegisterInput(t, inputs, specs["fake/input"]) mustRegisterChunker(t, chunkers, specs["generic"]) mustRegisterExtractor(t, extractors, specs["fake/extract"]) mustRegisterMerger(t, mergers, specs["appendorder"]) mustRegisterNormalizer(t, normalizers, specs["noop"]) mustRegisterValidator(t, validators, specs["fake/validator"]) mustRegisterValidator(t, validators, specs["fake/llm-validator"]) mustRegisterOutput(t, outputs, specs["json"]) codecs := pipeline.NewArtifactCodecRegistry() if err := pipeline.RegisterArtifactCodec(codecs, fakeArtifactCodec{}); err != nil { t.Fatalf("register artifact codec: %v", err) } return pipeline.ModuleCatalog{ Inputs: inputs, Chunkers: chunkers, ArtifactCodecs: codecs, Extractors: extractors, Mergers: mergers, Normalizers: normalizers, Validators: validators, ValidatorChains: pipeline.NewValidatorChainRegistry(), Outputs: outputs, } } func mustRegisterInput(t *testing.T, registry *pipeline.InputAdapterRegistry, spec pipeline.ModuleSpec) { t.Helper() if err := registry.RegisterWithSpec(spec, func() (contracts.InputAdapter, error) { return nil, nil }); err != nil { t.Fatalf("register input: %v", err) } } func mustRegisterChunker(t *testing.T, registry *pipeline.ChunkerRegistry, spec pipeline.ModuleSpec) { t.Helper() if err := registry.RegisterWithSpec(spec, func() (contracts.Chunker, error) { return nil, nil }); err != nil { t.Fatalf("register chunker: %v", err) } } func mustRegisterExtractor(t *testing.T, registry *pipeline.ExtractorRegistry, spec pipeline.ModuleSpec) { t.Helper() validateOptions := func(options map[string]any) error { if err := pipeline.RejectUnknownOptions(options, "temperature"); err != nil { return err } if value, ok := options["temperature"]; ok { if _, ok := value.(float64); !ok { return fmt.Errorf("temperature must be a number") } } return nil } if err := pipeline.RegisterExtractorBuilder[fakeArtifact](registry, spec, validateOptions, func(pipeline.BuildRequest) (contracts.Extractor[fakeArtifact], error) { return nil, nil }); err != nil { t.Fatalf("register extractor: %v", err) } } func mustRegisterMerger(t *testing.T, registry *pipeline.MergerRegistry, spec pipeline.ModuleSpec) { t.Helper() if err := pipeline.RegisterMerger[fakeArtifact](registry, spec, func() (contracts.Merger[fakeArtifact], error) { return nil, nil }); err != nil { t.Fatalf("register merger: %v", err) } } func mustRegisterNormalizer(t *testing.T, registry *pipeline.NormalizerRegistry, spec pipeline.ModuleSpec) { t.Helper() if err := pipeline.RegisterNormalizer[fakeArtifact](registry, spec, func() (contracts.Normalizer[fakeArtifact], error) { return nil, nil }); err != nil { t.Fatalf("register normalizer: %v", err) } } func mustRegisterValidator(t *testing.T, registry *pipeline.ValidatorRegistry, spec pipeline.ModuleSpec) { t.Helper() executionClass := contracts.ExecutionClassDeterministic if spec.Key == "fake/llm-validator" { executionClass = contracts.ExecutionClassLLMBacked } validatorSpec := pipeline.ValidatorSpec{Key: spec.Key, ExecutionClass: executionClass} if err := pipeline.RegisterTypedValidator[fakeArtifact](registry, fakeArtifactKind, validatorSpec, func() (contracts.TypedValidator[fakeArtifact], error) { return nil, nil }); err != nil { t.Fatalf("register validator: %v", err) } } const fakeArtifactKind contracts.ArtifactKind = "test/artifact" type fakeArtifact string type fakeArtifactCodec struct{} func (fakeArtifactCodec) Kind() contracts.ArtifactKind { return fakeArtifactKind } func (fakeArtifactCodec) Schema() contracts.ArtifactSchema { return contracts.ArtifactSchema{ID: "urn:notarius:test:artifact", Name: "Test artifact", Version: "1", JSONSchema: []byte(`{"type":"string"}`)} } func (fakeArtifactCodec) MediaType() string { return "application/json" } func (fakeArtifactCodec) EncodeCandidate(value fakeArtifact) ([]byte, error) { return []byte(fmt.Sprintf("%q", value)), nil } func (fakeArtifactCodec) Encode(value fakeArtifact) ([]byte, error) { return []byte(fmt.Sprintf("%q", value)), nil } func (fakeArtifactCodec) Decode(content []byte) (fakeArtifact, error) { if len(content) < 2 { return "", fmt.Errorf("invalid test artifact") } return fakeArtifact(content[1 : len(content)-1]), nil } func mustRegisterOutput(t *testing.T, registry *pipeline.OutputEncoderRegistry, spec pipeline.ModuleSpec) { t.Helper() if err := registry.RegisterWithSpec(spec, func() (contracts.OutputEncoder, error) { return nil, nil }); err != nil { t.Fatalf("register output: %v", err) } }