diff --git a/docs/cli.md b/docs/cli.md index c9f88e5..861e888 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -45,8 +45,9 @@ Flags: Defaults to `./notarius-output`. - `--diagnostics-dir path`: diagnostics work directory override for this invocation. -- `--llm-profile id`: override every effective LLM-capable module binding to - use one Scriptorium profile ID. +- `--llm-profile id`: override every effective LLM-capable pipeline module + binding to use one Scriptorium profile ID. Validator-specific profiles are + not overridden. - `--session-id id`: pass a stable prompt session identifier through LLM-backed module calls. - `--reference selector=path`: bind a reference path to a chunk, extractor, diff --git a/docs/config.md b/docs/config.md index e2f47d5..8cc8dda 100644 --- a/docs/config.md +++ b/docs/config.md @@ -131,8 +131,9 @@ Artifact lane fields: - `extract`: required module binding. - `merge`: optional module binding. Default module is `appendorder`. - `normalize`: optional module binding. Default module is `noop`. -- `validators`: reserved for future configurable validator chains. Non-empty - lists are rejected by current configuration validation. +- `validators`: deprecated lane-level validator list. Non-empty lists are + rejected; use `extract.validators`, `merge.validators`, or + `normalize.validators`. - `references`: optional compatibility alias for extractor reference bindings. Lane bindings override pipeline-level bindings for the same slot. @@ -234,12 +235,28 @@ Binding fields: - `options`: optional module-specific settings. - `references`: optional reference bindings. Supported only for `chunk`, `extract`, `merge`, and `normalize` bindings. `input` and `output` bindings - reject this field during validation. Validator bindings are reserved for a - future validator-chain feature and are rejected when configured. + reject this field during validation. +- `validators`: optional stage-local validator chain override. Supported only + for `chunk`, `extract`, `merge`, and `normalize` bindings. Omit the field to + use the production default chain; set `validators: []` to force an empty + chain; set a non-empty list to use exactly those validators in configured + order. + +Validator bindings use the same shorthand or object module-binding form, but +only these fields are supported: + +- `module`: validator key. +- `llm_profile`: optional Scriptorium profile ID for LLM-backed validators. +- `options`: optional validator-specific settings. + +Validator bindings reject `references`, `retries`, and nested `validators`. +During resolution, deterministic validators reject explicit `llm_profile` +values. The `--llm-profile` run flag overrides every effective LLM-capable module binding to use one Scriptorium profile ID: chunk, every selected lane extract, -merge, and normalize binding. +merge, and normalize binding. It does not override validator-specific +`llm_profile` values. ## Implemented Production Modules diff --git a/docs/internal/pipeline.md b/docs/internal/pipeline.md index 63c90c2..99be009 100644 --- a/docs/internal/pipeline.md +++ b/docs/internal/pipeline.md @@ -187,10 +187,10 @@ Runner-side raw validation chains receive the raw module output plus stage, lane, module, source, and chunk provenance. Empty raw validation chains approve output by default. -Pipeline-configured validator lists are not part of the current runner -contract. Non-empty configured validator lists are rejected during configuration -validation or resolved-run validation so they cannot appear in manifests without -executing. +Resolved validator chains come from central default mappings unless a +stage-local config override is set on `chunk`, lane `extract`, lane `merge`, or +lane `normalize`. Explicit empty overrides are valid and are recorded as empty +chains in manifests. Validator rejection is a non-fatal run outcome: the rejected output is recorded in `RunOutput.Rejected` and does not pass to the next stage. Validator execution diff --git a/internal/cli/run.go b/internal/cli/run.go index b66610a..cd19a27 100644 --- a/internal/cli/run.go +++ b/internal/cli/run.go @@ -495,6 +495,13 @@ func effectiveLLMProfileIDs(resolved pipeline.ResolvedPipeline) []string { add(lane.Merge) add(lane.Normalize) } + for _, chain := range resolved.ValidatorChains { + for _, validator := range chain.Validators { + if validator.ExecutionClass == contracts.ExecutionClassLLMBacked { + add(validator.Binding) + } + } + } ids := make([]string, 0, len(seen)) for id := range seen { ids = append(ids, id) diff --git a/internal/cli/run_test.go b/internal/cli/run_test.go index 297f15b..d4334f0 100644 --- a/internal/cli/run_test.go +++ b/internal/cli/run_test.go @@ -858,16 +858,32 @@ func TestEffectiveLLMProfileIDsUsesLLMCapableStagesOnly(t *testing.T) { Output: pipeline.ModuleBinding{LLMProfile: "output-profile"}, ArtifactLanes: []pipeline.ResolvedArtifactLane{ { - Extract: pipeline.ModuleBinding{LLMProfile: "extract-profile"}, - Merge: pipeline.ModuleBinding{LLMProfile: "merge-profile"}, - Normalize: pipeline.ModuleBinding{LLMProfile: "normalize-profile"}, - Validators: []pipeline.ModuleBinding{{LLMProfile: "validator-profile"}}, + Extract: pipeline.ModuleBinding{LLMProfile: "extract-profile"}, + Merge: pipeline.ModuleBinding{LLMProfile: "merge-profile"}, + Normalize: pipeline.ModuleBinding{LLMProfile: "normalize-profile"}, + }, + }, + ValidatorChains: []pipeline.ResolvedValidatorChain{ + { + Stage: pipeline.StageExtract, + LaneID: "events", + ModuleKey: "extract", + Validators: []pipeline.ResolvedValidator{ + { + Binding: pipeline.ModuleBinding{Module: "deterministic-validator", LLMProfile: "ignored-validator-profile"}, + ExecutionClass: contracts.ExecutionClassDeterministic, + }, + { + Binding: pipeline.ModuleBinding{Module: "llm-validator", LLMProfile: "validator-profile"}, + ExecutionClass: contracts.ExecutionClassLLMBacked, + }, + }, }, }, } got := effectiveLLMProfileIDs(resolved) - want := []string{"chunk-profile", "extract-profile", "merge-profile", "normalize-profile"} + want := []string{"chunk-profile", "extract-profile", "merge-profile", "normalize-profile", "validator-profile"} if !reflect.DeepEqual(got, want) { t.Fatalf("effectiveLLMProfileIDs() = %#v, want %#v", got, want) } diff --git a/internal/core/config/config.go b/internal/core/config/config.go index 5f1b115..9487ccb 100644 --- a/internal/core/config/config.go +++ b/internal/core/config/config.go @@ -97,6 +97,18 @@ func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding { out.Options = cloneOptions(in.Options) } out.References = cloneStringMap(in.References) + out.Validators = cloneValidatorOverride(in.Validators) + return out +} + +func cloneValidatorOverride(in pipeline.ValidatorOverride) pipeline.ValidatorOverride { + out := pipeline.ValidatorOverride{Set: in.Set} + if len(in.Validators) > 0 { + out.Validators = make([]pipeline.ModuleBinding, len(in.Validators)) + for i, binding := range in.Validators { + out.Validators[i] = cloneModuleBinding(binding) + } + } return out } diff --git a/internal/core/config/effective_config_test.go b/internal/core/config/effective_config_test.go index a127086..ea39f21 100644 --- a/internal/core/config/effective_config_test.go +++ b/internal/core/config/effective_config_test.go @@ -161,6 +161,12 @@ func TestResolveLLMProfileOverrideAppliesBeforeDigest(t *testing.T) { profile.Output.LLMProfile = "output-profile" lane := profile.Artifacts["events"] lane.Merge.LLMProfile = "merge-profile" + lane.Extract.Validators = pipeline.ValidatorOverride{ + Set: true, + Validators: []pipeline.ModuleBinding{ + {Module: "fake/llm-validator", LLMProfile: "validator-profile"}, + }, + } profile.Artifacts["events"] = lane cfg.Pipelines["example"] = profile @@ -195,9 +201,22 @@ func TestResolveLLMProfileOverrideAppliesBeforeDigest(t *testing.T) { if eventLane.Merge.LLMProfile != "runtime" { t.Fatalf("merge profile = %q, want runtime", eventLane.Merge.LLMProfile) } - if len(eventLane.Validators) != 0 { - t.Fatalf("validator profiles = %#v, want none", eventLane.Validators) + validatorChain := findEffectiveValidatorChain(effective.ResolvedPipeline.ValidatorChains, pipeline.StageExtract, "events", "fake/extract") + if validatorChain == nil || len(validatorChain.Validators) != 1 { + t.Fatalf("validator chain = %#v, want one extract validator", effective.ResolvedPipeline.ValidatorChains) } + if validatorChain.Validators[0].Binding.LLMProfile != "validator-profile" { + t.Fatalf("validator profile = %q, want original validator-profile", validatorChain.Validators[0].Binding.LLMProfile) + } +} + +func findEffectiveValidatorChain(chains []pipeline.ResolvedValidatorChain, stage pipeline.ModuleStage, laneID string, module string) *pipeline.ResolvedValidatorChain { + for i := range chains { + if chains[i].Stage == stage && chains[i].LaneID == laneID && chains[i].ModuleKey == module { + return &chains[i] + } + } + return nil } func llmCapableBindings(resolved pipeline.ResolvedPipeline) []pipeline.ModuleBinding { diff --git a/internal/core/config/file_config.go b/internal/core/config/file_config.go index cec535a..12899ff 100644 --- a/internal/core/config/file_config.go +++ b/internal/core/config/file_config.go @@ -56,6 +56,7 @@ type fileModuleBinding struct { Retries int Options map[string]any References map[string]string + Validators pipeline.ValidatorOverride } func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error { @@ -102,6 +103,16 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error { return err } b.References = references + case "validators": + b.Validators.Set = true + var validators []fileModuleBinding + if err := valueNode.Decode(&validators); err != nil { + return err + } + b.Validators.Validators = make([]pipeline.ModuleBinding, len(validators)) + for i, validator := range validators { + b.Validators.Validators[i] = validator.toPipelineBinding() + } default: return fmt.Errorf("field %s not found in module binding", keyNode.Value) } @@ -119,6 +130,7 @@ func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding { Retries: b.Retries, Options: cloneOptions(b.Options), References: normalizedStringMap(b.References), + Validators: b.Validators, } } diff --git a/internal/core/config/file_config_test.go b/internal/core/config/file_config_test.go index 12a9d3b..4e24691 100644 --- a/internal/core/config/file_config_test.go +++ b/internal/core/config/file_config_test.go @@ -300,6 +300,61 @@ pipelines: } } +func TestParseFileConfigStageLocalValidatorOverrides(t *testing.T) { + cfg := parseAndApplyConfig(t, ` +version: 2 +pipelines: + example: + input: fake/input + chunk: + module: generic + validators: [] + artifacts: + events: + extract: + module: fake/extract + validators: + - fake/validator + - module: fake/llm-validator + llm_profile: careful + options: + threshold: 0.7 + merge: + module: appendorder + validators: [] + normalize: + module: noop +`) + + profile := cfg.Pipelines["example"] + if !profile.Chunk.Validators.Set || len(profile.Chunk.Validators.Validators) != 0 { + t.Fatalf("chunk validator override = %#v, want explicit empty", profile.Chunk.Validators) + } + lane := profile.Artifacts["events"] + if !lane.Extract.Validators.Set { + t.Fatalf("extract validator override Set = false, want true") + } + validators := lane.Extract.Validators.Validators + if len(validators) != 2 { + t.Fatalf("extract validators = %#v, want two validators", validators) + } + if validators[0].Module != "fake/validator" { + t.Fatalf("first validator = %#v, want fake/validator", validators[0]) + } + if validators[1].Module != "fake/llm-validator" || validators[1].LLMProfile != "careful" { + t.Fatalf("second validator = %#v, want LLM validator with profile", validators[1]) + } + if validators[1].Options["threshold"] != 0.7 { + t.Fatalf("second validator options = %#v, want threshold", validators[1].Options) + } + if !lane.Merge.Validators.Set || len(lane.Merge.Validators.Validators) != 0 { + t.Fatalf("merge validator override = %#v, want explicit empty", lane.Merge.Validators) + } + if lane.Normalize.Validators.Set { + t.Fatalf("normalize validator override Set = true, want omitted") + } +} + func TestApplyFileConfigRejectsDuplicateTrimmedPipelineIDs(t *testing.T) { fileCfg, err := ParseFileConfigYAML([]byte(` version: 2 diff --git a/internal/core/config/redaction.go b/internal/core/config/redaction.go index b07619b..2a88f31 100644 --- a/internal/core/config/redaction.go +++ b/internal/core/config/redaction.go @@ -27,6 +27,12 @@ func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeli out.Chunk = cloneModuleBinding(in.Chunk) out.ChunkReferences = pipeline.CloneReferenceTarget(in.ChunkReferences) out.Output = cloneModuleBinding(in.Output) + if len(in.ValidatorChains) > 0 { + out.ValidatorChains = make([]pipeline.ResolvedValidatorChain, len(in.ValidatorChains)) + for i, chain := range in.ValidatorChains { + out.ValidatorChains[i] = cloneResolvedValidatorChain(chain) + } + } if len(in.ArtifactLanes) > 0 { out.ArtifactLanes = make([]pipeline.ResolvedArtifactLane, len(in.ArtifactLanes)) for i, lane := range in.ArtifactLanes { @@ -36,6 +42,20 @@ func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeli return out } +func cloneResolvedValidatorChain(in pipeline.ResolvedValidatorChain) pipeline.ResolvedValidatorChain { + out := in + if len(in.Validators) > 0 { + out.Validators = make([]pipeline.ResolvedValidator, len(in.Validators)) + for i, validator := range in.Validators { + out.Validators[i] = pipeline.ResolvedValidator{ + Binding: cloneModuleBinding(validator.Binding), + ExecutionClass: validator.ExecutionClass, + } + } + } + return out +} + func cloneResolvedArtifactLane(in pipeline.ResolvedArtifactLane) pipeline.ResolvedArtifactLane { out := in out.Extract = cloneModuleBinding(in.Extract) diff --git a/internal/core/config/validation.go b/internal/core/config/validation.go index 51aaeae..f76a120 100644 --- a/internal/core/config/validation.go +++ b/internal/core/config/validation.go @@ -85,7 +85,7 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile) erro return err } if len(lane.Validators) > 0 { - return fmt.Errorf("pipeline %q lane %q validators are not supported by the current raw validation runner", id, laneID) + return fmt.Errorf("pipeline %q lane %q validators are not supported at artifact lane level; use extract.validators, merge.validators, or normalize.validators", id, laneID) } } } @@ -108,6 +108,9 @@ func validateBinding( } return fmt.Errorf("pipeline %q %s retries must be greater than or equal to zero", pipelineID, slot) } + if err := validateValidatorOverride(pipelineID, laneID, slot, binding.Validators); err != nil { + return err + } if len(binding.References) == 0 { return nil } @@ -120,6 +123,36 @@ func validateBinding( return validateReferenceMapForContext(pipelineID, laneID, slot, binding.References) } +func validateValidatorOverride(pipelineID string, laneID string, slot string, override pipeline.ValidatorOverride) error { + if !override.Set { + return nil + } + switch slot { + case "chunk", "extract", "merge", "normalize": + default: + return fmt.Errorf("%s validators are not supported", referenceContext(pipelineID, laneID, slot)) + } + for i, validator := range override.Validators { + context := fmt.Sprintf("%s validators[%d]", referenceContext(pipelineID, laneID, slot), i) + if strings.TrimSpace(validator.Module) == "" { + return fmt.Errorf("%s module must not be empty", context) + } + if len(validator.References) > 0 { + return fmt.Errorf("%s references are not supported", context) + } + if validator.Validators.Set { + return fmt.Errorf("%s nested validators are not supported", context) + } + if validator.Retries != 0 { + return fmt.Errorf("%s retries are not supported", context) + } + if validator.LLMProfile != "" && strings.TrimSpace(validator.LLMProfile) == "" { + return fmt.Errorf("%s llm_profile must not be empty when set", context) + } + } + return nil +} + func validateReferenceMap(pipelineID string, laneID string, references map[string]string) error { return validateReferenceMapForContext(pipelineID, laneID, "", references) } diff --git a/internal/core/config/validation_test.go b/internal/core/config/validation_test.go index f350455..01b6143 100644 --- a/internal/core/config/validation_test.go +++ b/internal/core/config/validation_test.go @@ -341,13 +341,82 @@ func TestValidateRejectsConfiguredValidators(t *testing.T) { if err == nil { t.Fatal("Validate() error = nil, want configured validators error") } - for _, want := range []string{"example", "events", "validators", "not supported"} { + for _, want := range []string{"example", "events", "validators", "extract.validators", "merge.validators", "normalize.validators"} { if !strings.Contains(err.Error(), want) { t.Fatalf("Validate() error = %q, want substring %q", err.Error(), want) } } } +func TestValidateAcceptsStageLocalValidatorOverrides(t *testing.T) { + cfg := validConfig() + profile := cfg.Pipelines["example"] + profile.Chunk.Validators = pipeline.ValidatorOverride{Set: true} + lane := profile.Artifacts["events"] + lane.Extract.Validators = pipeline.ValidatorOverride{ + Set: true, + Validators: []pipeline.ModuleBinding{ + pipeline.Binding("fake/validator"), + {Module: "fake/llm-validator", LLMProfile: "careful", Options: map[string]any{"threshold": 0.7}}, + }, + } + lane.Merge.Validators = pipeline.ValidatorOverride{Set: true} + profile.Artifacts["events"] = lane + cfg.Pipelines["example"] = profile + + if err := cfg.Validate(); err != nil { + t.Fatalf("Validate() error = %v, want nil", err) + } +} + +func TestValidateRejectsInvalidValidatorBindings(t *testing.T) { + tests := []struct { + name string + binding pipeline.ModuleBinding + want string + }{ + { + name: "empty module", + binding: pipeline.ModuleBinding{}, + want: "module must not be empty", + }, + { + name: "references", + binding: pipeline.ModuleBinding{Module: "fake/validator", References: map[string]string{"roster": "./roster.txt"}}, + want: "references are not supported", + }, + { + name: "nested validators", + binding: pipeline.ModuleBinding{Module: "fake/validator", Validators: pipeline.ValidatorOverride{Set: true}}, + want: "nested validators are not supported", + }, + { + name: "retries", + binding: pipeline.ModuleBinding{Module: "fake/validator", Retries: 1}, + want: "retries are not supported", + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cfg := validConfig() + profile := cfg.Pipelines["example"] + lane := profile.Artifacts["events"] + lane.Extract.Validators = pipeline.ValidatorOverride{ + Set: true, + Validators: []pipeline.ModuleBinding{test.binding}, + } + profile.Artifacts["events"] = lane + cfg.Pipelines["example"] = profile + + err := cfg.Validate() + if err == nil || !strings.Contains(err.Error(), test.want) { + t.Fatalf("Validate() error = %v, want %q", err, test.want) + } + }) + } +} + func validConfig() Config { cfg := Default() cfg.Pipelines["example"] = pipeline.PipelineProfile{ @@ -402,6 +471,12 @@ func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.Module Requires: []string{"normalized"}, Provides: []string{"validated"}, }, + "fake/llm-validator": { + Key: "fake/llm-validator", + Stage: pipeline.StageValidate, + Requires: []string{"normalized"}, + Provides: []string{"validated"}, + }, "json": { Key: "json", Stage: pipeline.StageOutput, @@ -426,6 +501,7 @@ func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.Module mustRegisterMerger(t, mergers, specs["appendorder"]) mustRegisterNormalizer(t, normalizers, specs["noop"]) mustRegisterValidator(t, validators, specs["fake/validator"]) + mustRegisterValidator(t, validators, specs["fake/llm-validator"]) mustRegisterOutput(t, outputs, specs["json"]) return pipeline.ModuleCatalog{ @@ -477,7 +553,11 @@ func mustRegisterNormalizer(t *testing.T, registry *pipeline.NormalizerRegistry, func mustRegisterValidator(t *testing.T, registry *pipeline.ValidatorRegistry, spec pipeline.ModuleSpec) { t.Helper() - validatorSpec := pipeline.ValidatorSpec{Key: spec.Key, ExecutionClass: contracts.ExecutionClassDeterministic} + executionClass := contracts.ExecutionClassDeterministic + if spec.Key == "fake/llm-validator" { + executionClass = contracts.ExecutionClassLLMBacked + } + validatorSpec := pipeline.ValidatorSpec{Key: spec.Key, ExecutionClass: executionClass} if err := registry.RegisterWithSpec(validatorSpec, func() (contracts.Validator, error) { return nil, nil }); err != nil { t.Fatalf("register validator: %v", err) } diff --git a/internal/framework/pipeline/profile.go b/internal/framework/pipeline/profile.go index f7f2b06..e9c8454 100644 --- a/internal/framework/pipeline/profile.go +++ b/internal/framework/pipeline/profile.go @@ -25,6 +25,35 @@ type ModuleBinding struct { Retries int `json:"retries,omitempty"` Options map[string]any `json:"options,omitempty"` References map[string]string `json:"references,omitempty"` + Validators ValidatorOverride `json:"validators,omitempty"` +} + +type ValidatorOverride struct { + Set bool `json:"set,omitempty"` + Validators []ModuleBinding `json:"validators,omitempty"` +} + +func (binding ModuleBinding) MarshalJSON() ([]byte, error) { + type moduleBindingJSON struct { + Module string `json:"module"` + LLMProfile string `json:"llm_profile,omitempty"` + Retries int `json:"retries,omitempty"` + Options map[string]any `json:"options,omitempty"` + References map[string]string `json:"references,omitempty"` + Validators *[]ModuleBinding `json:"validators,omitempty"` + } + out := moduleBindingJSON{ + Module: binding.Module, + LLMProfile: binding.LLMProfile, + Retries: binding.Retries, + Options: binding.Options, + References: binding.References, + } + if binding.Validators.Set { + validators := cloneModuleBindings(binding.Validators.Validators) + out.Validators = &validators + } + return json.Marshal(out) } type ArtifactLaneProfile struct { @@ -186,7 +215,7 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences), Output: resolveBinding(profile.Output, DefaultOutputModule), } - chunkValidatorChain, err := resolveValidatorChain(pipelineID, "", StageChunk, chunk.Module, catalog) + chunkValidatorChain, err := resolveValidatorChain(pipelineID, "", StageChunk, chunk.Module, chunk.Validators, catalog) if err != nil { return ResolvedPipeline{}, err } @@ -316,15 +345,15 @@ func resolveArtifactLane( return ResolvedArtifactLane{}, nil, nil, configuredValidatorsError(pipelineID, laneID) } - extractValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageExtract, lane.Extract.Module, catalog) + extractValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageExtract, lane.Extract.Module, lane.Extract.Validators, catalog) if err != nil { return ResolvedArtifactLane{}, nil, nil, err } - mergeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageMerge, lane.Merge.Module, catalog) + mergeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageMerge, lane.Merge.Module, lane.Merge.Validators, catalog) if err != nil { return ResolvedArtifactLane{}, nil, nil, err } - normalizeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageNormalize, lane.Normalize.Module, catalog) + normalizeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageNormalize, lane.Normalize.Module, lane.Normalize.Validators, catalog) if err != nil { return ResolvedArtifactLane{}, nil, nil, err } @@ -334,10 +363,10 @@ func resolveArtifactLane( } func configuredValidatorsError(pipelineID string, laneID string) error { - return fmt.Errorf("pipeline %q lane %q configured validators are not supported by the current raw validation runner", pipelineID, laneID) + return fmt.Errorf("pipeline %q lane %q validators are not supported at artifact lane level; use extract.validators, merge.validators, or normalize.validators", pipelineID, laneID) } -func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage, module string, catalog ModuleCatalog) (ResolvedValidatorChain, error) { +func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage, module string, override ValidatorOverride, catalog ModuleCatalog) (ResolvedValidatorChain, error) { chain := ResolvedValidatorChain{ Stage: stage, LaneID: strings.TrimSpace(laneID), @@ -352,10 +381,12 @@ func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage, return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q validator chain stage %q is not supported", pipelineID, stage) } - if catalog.ValidatorChains == nil { - return chain, nil + var bindings []ModuleBinding + if override.Set { + bindings = cloneModuleBindings(override.Validators) + } else if catalog.ValidatorChains != nil { + bindings = catalog.ValidatorChains.Validators(stage, chain.ModuleKey) } - bindings := catalog.ValidatorChains.Validators(stage, chain.ModuleKey) if len(bindings) == 0 { return chain, nil } @@ -368,6 +399,9 @@ func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage, if !ok { return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q %s validator chain for module %q references unknown validator %q", pipelineID, stage, chain.ModuleKey, validator.Module) } + if strings.TrimSpace(validator.LLMProfile) != "" && spec.ExecutionClass != contracts.ExecutionClassLLMBacked { + return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q %s validator chain for module %q assigns llm_profile to deterministic validator %q", pipelineID, stage, chain.ModuleKey, validator.Module) + } chain.Validators = append(chain.Validators, ResolvedValidator{ Binding: cloneModuleBinding(validator), ExecutionClass: spec.ExecutionClass, @@ -728,6 +762,7 @@ func resolveBinding(binding ModuleBinding, defaultModule string) ModuleBinding { Retries: binding.Retries, Options: cloneOptions(binding.Options), References: normalizeReferenceMap(binding.References), + Validators: cloneValidatorOverride(binding.Validators), } } diff --git a/internal/framework/pipeline/profile_test.go b/internal/framework/pipeline/profile_test.go index 9e63cea..9570b00 100644 --- a/internal/framework/pipeline/profile_test.go +++ b/internal/framework/pipeline/profile_test.go @@ -182,6 +182,143 @@ func TestResolvePipelineRejectsUnknownDefaultValidator(t *testing.T) { } } +func TestResolvePipelineValidatorOverrideReplacesDefaultChain(t *testing.T) { + catalog := newProfileCatalog(t) + registerProfileValidatorSpec(t, catalog, ValidatorSpec{Key: "second-validator", ExecutionClass: contracts.ExecutionClassLLMBacked}) + if err := catalog.ValidatorChains.Register(ValidatorChainMapping{ + Stage: StageExtract, + Module: "event-extractor", + Validators: []ModuleBinding{Binding("grounded")}, + }); err != nil { + t.Fatalf("register validator chain: %v", err) + } + + profile := PipelineProfile{ + ID: "validated", + Input: Binding("text"), + Artifacts: map[string]ArtifactLaneProfile{ + "events": { + Extract: ModuleBinding{ + Module: "event-extractor", + Validators: ValidatorOverride{ + Set: true, + Validators: []ModuleBinding{ + {Module: "second-validator", LLMProfile: "careful"}, + Binding("grounded"), + }, + }, + }, + }, + }, + } + + resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog) + if err != nil { + t.Fatalf("ResolvePipeline() error = %v, want nil", err) + } + + extractChain := findResolvedValidatorChain(resolved.ValidatorChains, StageExtract, "events", "event-extractor") + if extractChain == nil { + t.Fatal("extract validator chain not found") + } + if len(extractChain.Validators) != 2 { + t.Fatalf("extract validators = %#v, want explicit two-validator override", extractChain.Validators) + } + if extractChain.Validators[0].Binding.Module != "second-validator" || extractChain.Validators[0].Binding.LLMProfile != "careful" { + t.Fatalf("first validator = %#v, want explicit LLM-backed validator first", extractChain.Validators[0]) + } + if extractChain.Validators[1].Binding.Module != "grounded" { + t.Fatalf("second validator = %#v, want grounded second", extractChain.Validators[1]) + } +} + +func TestResolvePipelineExplicitEmptyValidatorOverrideSuppressesDefaultChain(t *testing.T) { + catalog := newProfileCatalog(t) + if err := catalog.ValidatorChains.Register(ValidatorChainMapping{ + Stage: StageExtract, + Module: "event-extractor", + Validators: []ModuleBinding{Binding("grounded")}, + }); err != nil { + t.Fatalf("register validator chain: %v", err) + } + + profile := PipelineProfile{ + ID: "validated", + Input: Binding("text"), + Artifacts: map[string]ArtifactLaneProfile{ + "events": { + Extract: ModuleBinding{ + Module: "event-extractor", + Validators: ValidatorOverride{Set: true}, + }, + }, + }, + } + + resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog) + if err != nil { + t.Fatalf("ResolvePipeline() error = %v, want nil", err) + } + + extractChain := findResolvedValidatorChain(resolved.ValidatorChains, StageExtract, "events", "event-extractor") + if extractChain == nil { + t.Fatal("extract validator chain not found") + } + if len(extractChain.Validators) != 0 { + t.Fatalf("extract validators = %#v, want explicit empty override", extractChain.Validators) + } +} + +func TestResolvePipelineRejectsUnknownOverrideValidator(t *testing.T) { + _, err := ResolvePipeline(PipelineProfile{ + ID: "validated", + Input: Binding("text"), + Artifacts: map[string]ArtifactLaneProfile{ + "events": { + Extract: ModuleBinding{ + Module: "event-extractor", + Validators: ValidatorOverride{ + Set: true, + Validators: []ModuleBinding{Binding("missing-validator")}, + }, + }, + }, + }, + }, ResolveOptions{}, newProfileCatalog(t)) + if err == nil { + t.Fatal("ResolvePipeline() error = nil, want unknown validator error") + } + if !strings.Contains(err.Error(), "missing-validator") { + t.Fatalf("ResolvePipeline() error = %q, want missing validator context", err.Error()) + } +} + +func TestResolvePipelineRejectsLLMProfileForDeterministicValidator(t *testing.T) { + _, err := ResolvePipeline(PipelineProfile{ + ID: "validated", + Input: Binding("text"), + Artifacts: map[string]ArtifactLaneProfile{ + "events": { + Extract: ModuleBinding{ + Module: "event-extractor", + Validators: ValidatorOverride{ + Set: true, + Validators: []ModuleBinding{ + {Module: "grounded", LLMProfile: "careful"}, + }, + }, + }, + }, + }, + }, ResolveOptions{}, newProfileCatalog(t)) + if err == nil { + t.Fatal("ResolvePipeline() error = nil, want deterministic validator profile error") + } + if !strings.Contains(err.Error(), "grounded") || !strings.Contains(err.Error(), "llm_profile") { + t.Fatalf("ResolvePipeline() error = %q, want validator profile context", err.Error()) + } +} + func TestResolvePipelineSelectsOnlyRequestedLanes(t *testing.T) { profile := multiLaneProfile() resolved, err := ResolvePipeline(profile, ResolveOptions{Only: []string{" summaries ", "events", "summaries"}}, newProfileCatalog(t)) @@ -850,7 +987,7 @@ func TestResolvePipelineRejectsConfiguredValidators(t *testing.T) { if err == nil { t.Fatal("ResolvePipeline() error = nil, want error") } - assertErrorContains(t, err, "baseline", "events", "configured validators", "not supported") + assertErrorContains(t, err, "baseline", "events", "validators", "extract.validators") } func TestResolvePipelineOrdersLanesDeterministically(t *testing.T) { @@ -1139,6 +1276,13 @@ func registerProfileSpecs(t *testing.T, catalog ModuleCatalog, specs ...ModuleSp } } +func registerProfileValidatorSpec(t *testing.T, catalog ModuleCatalog, spec ValidatorSpec) { + t.Helper() + if err := catalog.Validators.RegisterWithSpec(spec, profileValidatorConstructor(spec.Key)); err != nil { + t.Fatalf("register validator spec %#v: %v", spec, err) + } +} + func profileInputConstructor(key string) InputAdapterConstructor { return func() (contracts.InputAdapter, error) { return profileInputAdapter{key: key}, nil diff --git a/internal/framework/pipeline/runner.go b/internal/framework/pipeline/runner.go index cee362b..c1c734a 100644 --- a/internal/framework/pipeline/runner.go +++ b/internal/framework/pipeline/runner.go @@ -631,7 +631,7 @@ func validateRunInput(input RunInput) error { return fmt.Errorf("resolved pipeline lane %q normalize module must not be empty", lane.ID) } if len(lane.Validators) > 0 { - return fmt.Errorf("resolved pipeline lane %q configured validators are not supported by the current raw validation runner", lane.ID) + return fmt.Errorf("resolved pipeline lane %q validators are not supported at artifact lane level; use extract.validators, merge.validators, or normalize.validators", lane.ID) } } return nil diff --git a/internal/framework/pipeline/runner_test.go b/internal/framework/pipeline/runner_test.go index 91080f2..f175e42 100644 --- a/internal/framework/pipeline/runner_test.go +++ b/internal/framework/pipeline/runner_test.go @@ -1081,7 +1081,7 @@ func TestRunRejectsConfiguredValidators(t *testing.T) { _, err := New(newRunnerRegistries(t, defaultRunnerModules())).Run(context.Background(), RunInput{ Pipeline: resolvedPipelineWithValidators("configured", "second-validator"), }) - assertRunError(t, err, "configured validators") + assertRunError(t, err, "extract.validators") } func TestRunCollectsStageWarnings(t *testing.T) { diff --git a/internal/framework/pipeline/validator_chain_registry.go b/internal/framework/pipeline/validator_chain_registry.go index 0e8882e..fb6c0fe 100644 --- a/internal/framework/pipeline/validator_chain_registry.go +++ b/internal/framework/pipeline/validator_chain_registry.go @@ -98,5 +98,13 @@ func cloneModuleBinding(binding ModuleBinding) ModuleBinding { } binding.References = references } + binding.Validators = cloneValidatorOverride(binding.Validators) return binding } + +func cloneValidatorOverride(override ValidatorOverride) ValidatorOverride { + return ValidatorOverride{ + Set: override.Set, + Validators: cloneModuleBindings(override.Validators), + } +}