Add validator chain config overrides

This commit is contained in:
2026-07-07 21:21:12 +00:00
parent d593bfee0a
commit 666b4bf801
17 changed files with 492 additions and 33 deletions

View File

@@ -45,8 +45,9 @@ Flags:
Defaults to `./notarius-output`. Defaults to `./notarius-output`.
- `--diagnostics-dir path`: diagnostics work directory override for this - `--diagnostics-dir path`: diagnostics work directory override for this
invocation. invocation.
- `--llm-profile id`: override every effective LLM-capable module binding to - `--llm-profile id`: override every effective LLM-capable pipeline module
use one Scriptorium profile ID. binding to use one Scriptorium profile ID. Validator-specific profiles are
not overridden.
- `--session-id id`: pass a stable prompt session identifier through LLM-backed - `--session-id id`: pass a stable prompt session identifier through LLM-backed
module calls. module calls.
- `--reference selector=path`: bind a reference path to a chunk, extractor, - `--reference selector=path`: bind a reference path to a chunk, extractor,

View File

@@ -131,8 +131,9 @@ Artifact lane fields:
- `extract`: required module binding. - `extract`: required module binding.
- `merge`: optional module binding. Default module is `appendorder`. - `merge`: optional module binding. Default module is `appendorder`.
- `normalize`: optional module binding. Default module is `noop`. - `normalize`: optional module binding. Default module is `noop`.
- `validators`: reserved for future configurable validator chains. Non-empty - `validators`: deprecated lane-level validator list. Non-empty lists are
lists are rejected by current configuration validation. rejected; use `extract.validators`, `merge.validators`, or
`normalize.validators`.
- `references`: optional compatibility alias for extractor reference bindings. - `references`: optional compatibility alias for extractor reference bindings.
Lane bindings override pipeline-level bindings for the same slot. Lane bindings override pipeline-level bindings for the same slot.
@@ -234,12 +235,28 @@ Binding fields:
- `options`: optional module-specific settings. - `options`: optional module-specific settings.
- `references`: optional reference bindings. Supported only for `chunk`, - `references`: optional reference bindings. Supported only for `chunk`,
`extract`, `merge`, and `normalize` bindings. `input` and `output` bindings `extract`, `merge`, and `normalize` bindings. `input` and `output` bindings
reject this field during validation. Validator bindings are reserved for a reject this field during validation.
future validator-chain feature and are rejected when configured. - `validators`: optional stage-local validator chain override. Supported only
for `chunk`, `extract`, `merge`, and `normalize` bindings. Omit the field to
use the production default chain; set `validators: []` to force an empty
chain; set a non-empty list to use exactly those validators in configured
order.
Validator bindings use the same shorthand or object module-binding form, but
only these fields are supported:
- `module`: validator key.
- `llm_profile`: optional Scriptorium profile ID for LLM-backed validators.
- `options`: optional validator-specific settings.
Validator bindings reject `references`, `retries`, and nested `validators`.
During resolution, deterministic validators reject explicit `llm_profile`
values.
The `--llm-profile` run flag overrides every effective LLM-capable module The `--llm-profile` run flag overrides every effective LLM-capable module
binding to use one Scriptorium profile ID: chunk, every selected lane extract, binding to use one Scriptorium profile ID: chunk, every selected lane extract,
merge, and normalize binding. merge, and normalize binding. It does not override validator-specific
`llm_profile` values.
## Implemented Production Modules ## Implemented Production Modules

View File

@@ -187,10 +187,10 @@ Runner-side raw validation chains receive the raw module output plus
stage, lane, module, source, and chunk provenance. Empty raw validation chains stage, lane, module, source, and chunk provenance. Empty raw validation chains
approve output by default. approve output by default.
Pipeline-configured validator lists are not part of the current runner Resolved validator chains come from central default mappings unless a
contract. Non-empty configured validator lists are rejected during configuration stage-local config override is set on `chunk`, lane `extract`, lane `merge`, or
validation or resolved-run validation so they cannot appear in manifests without lane `normalize`. Explicit empty overrides are valid and are recorded as empty
executing. chains in manifests.
Validator rejection is a non-fatal run outcome: the rejected output is recorded Validator rejection is a non-fatal run outcome: the rejected output is recorded
in `RunOutput.Rejected` and does not pass to the next stage. Validator execution in `RunOutput.Rejected` and does not pass to the next stage. Validator execution

View File

@@ -495,6 +495,13 @@ func effectiveLLMProfileIDs(resolved pipeline.ResolvedPipeline) []string {
add(lane.Merge) add(lane.Merge)
add(lane.Normalize) add(lane.Normalize)
} }
for _, chain := range resolved.ValidatorChains {
for _, validator := range chain.Validators {
if validator.ExecutionClass == contracts.ExecutionClassLLMBacked {
add(validator.Binding)
}
}
}
ids := make([]string, 0, len(seen)) ids := make([]string, 0, len(seen))
for id := range seen { for id := range seen {
ids = append(ids, id) ids = append(ids, id)

View File

@@ -858,16 +858,32 @@ func TestEffectiveLLMProfileIDsUsesLLMCapableStagesOnly(t *testing.T) {
Output: pipeline.ModuleBinding{LLMProfile: "output-profile"}, Output: pipeline.ModuleBinding{LLMProfile: "output-profile"},
ArtifactLanes: []pipeline.ResolvedArtifactLane{ ArtifactLanes: []pipeline.ResolvedArtifactLane{
{ {
Extract: pipeline.ModuleBinding{LLMProfile: "extract-profile"}, Extract: pipeline.ModuleBinding{LLMProfile: "extract-profile"},
Merge: pipeline.ModuleBinding{LLMProfile: "merge-profile"}, Merge: pipeline.ModuleBinding{LLMProfile: "merge-profile"},
Normalize: pipeline.ModuleBinding{LLMProfile: "normalize-profile"}, Normalize: pipeline.ModuleBinding{LLMProfile: "normalize-profile"},
Validators: []pipeline.ModuleBinding{{LLMProfile: "validator-profile"}}, },
},
ValidatorChains: []pipeline.ResolvedValidatorChain{
{
Stage: pipeline.StageExtract,
LaneID: "events",
ModuleKey: "extract",
Validators: []pipeline.ResolvedValidator{
{
Binding: pipeline.ModuleBinding{Module: "deterministic-validator", LLMProfile: "ignored-validator-profile"},
ExecutionClass: contracts.ExecutionClassDeterministic,
},
{
Binding: pipeline.ModuleBinding{Module: "llm-validator", LLMProfile: "validator-profile"},
ExecutionClass: contracts.ExecutionClassLLMBacked,
},
},
}, },
}, },
} }
got := effectiveLLMProfileIDs(resolved) got := effectiveLLMProfileIDs(resolved)
want := []string{"chunk-profile", "extract-profile", "merge-profile", "normalize-profile"} want := []string{"chunk-profile", "extract-profile", "merge-profile", "normalize-profile", "validator-profile"}
if !reflect.DeepEqual(got, want) { if !reflect.DeepEqual(got, want) {
t.Fatalf("effectiveLLMProfileIDs() = %#v, want %#v", got, want) t.Fatalf("effectiveLLMProfileIDs() = %#v, want %#v", got, want)
} }

View File

@@ -97,6 +97,18 @@ func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
out.Options = cloneOptions(in.Options) out.Options = cloneOptions(in.Options)
} }
out.References = cloneStringMap(in.References) out.References = cloneStringMap(in.References)
out.Validators = cloneValidatorOverride(in.Validators)
return out
}
func cloneValidatorOverride(in pipeline.ValidatorOverride) pipeline.ValidatorOverride {
out := pipeline.ValidatorOverride{Set: in.Set}
if len(in.Validators) > 0 {
out.Validators = make([]pipeline.ModuleBinding, len(in.Validators))
for i, binding := range in.Validators {
out.Validators[i] = cloneModuleBinding(binding)
}
}
return out return out
} }

View File

@@ -161,6 +161,12 @@ func TestResolveLLMProfileOverrideAppliesBeforeDigest(t *testing.T) {
profile.Output.LLMProfile = "output-profile" profile.Output.LLMProfile = "output-profile"
lane := profile.Artifacts["events"] lane := profile.Artifacts["events"]
lane.Merge.LLMProfile = "merge-profile" lane.Merge.LLMProfile = "merge-profile"
lane.Extract.Validators = pipeline.ValidatorOverride{
Set: true,
Validators: []pipeline.ModuleBinding{
{Module: "fake/llm-validator", LLMProfile: "validator-profile"},
},
}
profile.Artifacts["events"] = lane profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile cfg.Pipelines["example"] = profile
@@ -195,9 +201,22 @@ func TestResolveLLMProfileOverrideAppliesBeforeDigest(t *testing.T) {
if eventLane.Merge.LLMProfile != "runtime" { if eventLane.Merge.LLMProfile != "runtime" {
t.Fatalf("merge profile = %q, want runtime", eventLane.Merge.LLMProfile) t.Fatalf("merge profile = %q, want runtime", eventLane.Merge.LLMProfile)
} }
if len(eventLane.Validators) != 0 { validatorChain := findEffectiveValidatorChain(effective.ResolvedPipeline.ValidatorChains, pipeline.StageExtract, "events", "fake/extract")
t.Fatalf("validator profiles = %#v, want none", eventLane.Validators) if validatorChain == nil || len(validatorChain.Validators) != 1 {
t.Fatalf("validator chain = %#v, want one extract validator", effective.ResolvedPipeline.ValidatorChains)
} }
if validatorChain.Validators[0].Binding.LLMProfile != "validator-profile" {
t.Fatalf("validator profile = %q, want original validator-profile", validatorChain.Validators[0].Binding.LLMProfile)
}
}
func findEffectiveValidatorChain(chains []pipeline.ResolvedValidatorChain, stage pipeline.ModuleStage, laneID string, module string) *pipeline.ResolvedValidatorChain {
for i := range chains {
if chains[i].Stage == stage && chains[i].LaneID == laneID && chains[i].ModuleKey == module {
return &chains[i]
}
}
return nil
} }
func llmCapableBindings(resolved pipeline.ResolvedPipeline) []pipeline.ModuleBinding { func llmCapableBindings(resolved pipeline.ResolvedPipeline) []pipeline.ModuleBinding {

View File

@@ -56,6 +56,7 @@ type fileModuleBinding struct {
Retries int Retries int
Options map[string]any Options map[string]any
References map[string]string References map[string]string
Validators pipeline.ValidatorOverride
} }
func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error { func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
@@ -102,6 +103,16 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
return err return err
} }
b.References = references b.References = references
case "validators":
b.Validators.Set = true
var validators []fileModuleBinding
if err := valueNode.Decode(&validators); err != nil {
return err
}
b.Validators.Validators = make([]pipeline.ModuleBinding, len(validators))
for i, validator := range validators {
b.Validators.Validators[i] = validator.toPipelineBinding()
}
default: default:
return fmt.Errorf("field %s not found in module binding", keyNode.Value) return fmt.Errorf("field %s not found in module binding", keyNode.Value)
} }
@@ -119,6 +130,7 @@ func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
Retries: b.Retries, Retries: b.Retries,
Options: cloneOptions(b.Options), Options: cloneOptions(b.Options),
References: normalizedStringMap(b.References), References: normalizedStringMap(b.References),
Validators: b.Validators,
} }
} }

View File

@@ -300,6 +300,61 @@ pipelines:
} }
} }
func TestParseFileConfigStageLocalValidatorOverrides(t *testing.T) {
cfg := parseAndApplyConfig(t, `
version: 2
pipelines:
example:
input: fake/input
chunk:
module: generic
validators: []
artifacts:
events:
extract:
module: fake/extract
validators:
- fake/validator
- module: fake/llm-validator
llm_profile: careful
options:
threshold: 0.7
merge:
module: appendorder
validators: []
normalize:
module: noop
`)
profile := cfg.Pipelines["example"]
if !profile.Chunk.Validators.Set || len(profile.Chunk.Validators.Validators) != 0 {
t.Fatalf("chunk validator override = %#v, want explicit empty", profile.Chunk.Validators)
}
lane := profile.Artifacts["events"]
if !lane.Extract.Validators.Set {
t.Fatalf("extract validator override Set = false, want true")
}
validators := lane.Extract.Validators.Validators
if len(validators) != 2 {
t.Fatalf("extract validators = %#v, want two validators", validators)
}
if validators[0].Module != "fake/validator" {
t.Fatalf("first validator = %#v, want fake/validator", validators[0])
}
if validators[1].Module != "fake/llm-validator" || validators[1].LLMProfile != "careful" {
t.Fatalf("second validator = %#v, want LLM validator with profile", validators[1])
}
if validators[1].Options["threshold"] != 0.7 {
t.Fatalf("second validator options = %#v, want threshold", validators[1].Options)
}
if !lane.Merge.Validators.Set || len(lane.Merge.Validators.Validators) != 0 {
t.Fatalf("merge validator override = %#v, want explicit empty", lane.Merge.Validators)
}
if lane.Normalize.Validators.Set {
t.Fatalf("normalize validator override Set = true, want omitted")
}
}
func TestApplyFileConfigRejectsDuplicateTrimmedPipelineIDs(t *testing.T) { func TestApplyFileConfigRejectsDuplicateTrimmedPipelineIDs(t *testing.T) {
fileCfg, err := ParseFileConfigYAML([]byte(` fileCfg, err := ParseFileConfigYAML([]byte(`
version: 2 version: 2

View File

@@ -27,6 +27,12 @@ func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeli
out.Chunk = cloneModuleBinding(in.Chunk) out.Chunk = cloneModuleBinding(in.Chunk)
out.ChunkReferences = pipeline.CloneReferenceTarget(in.ChunkReferences) out.ChunkReferences = pipeline.CloneReferenceTarget(in.ChunkReferences)
out.Output = cloneModuleBinding(in.Output) out.Output = cloneModuleBinding(in.Output)
if len(in.ValidatorChains) > 0 {
out.ValidatorChains = make([]pipeline.ResolvedValidatorChain, len(in.ValidatorChains))
for i, chain := range in.ValidatorChains {
out.ValidatorChains[i] = cloneResolvedValidatorChain(chain)
}
}
if len(in.ArtifactLanes) > 0 { if len(in.ArtifactLanes) > 0 {
out.ArtifactLanes = make([]pipeline.ResolvedArtifactLane, len(in.ArtifactLanes)) out.ArtifactLanes = make([]pipeline.ResolvedArtifactLane, len(in.ArtifactLanes))
for i, lane := range in.ArtifactLanes { for i, lane := range in.ArtifactLanes {
@@ -36,6 +42,20 @@ func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeli
return out return out
} }
func cloneResolvedValidatorChain(in pipeline.ResolvedValidatorChain) pipeline.ResolvedValidatorChain {
out := in
if len(in.Validators) > 0 {
out.Validators = make([]pipeline.ResolvedValidator, len(in.Validators))
for i, validator := range in.Validators {
out.Validators[i] = pipeline.ResolvedValidator{
Binding: cloneModuleBinding(validator.Binding),
ExecutionClass: validator.ExecutionClass,
}
}
}
return out
}
func cloneResolvedArtifactLane(in pipeline.ResolvedArtifactLane) pipeline.ResolvedArtifactLane { func cloneResolvedArtifactLane(in pipeline.ResolvedArtifactLane) pipeline.ResolvedArtifactLane {
out := in out := in
out.Extract = cloneModuleBinding(in.Extract) out.Extract = cloneModuleBinding(in.Extract)

View File

@@ -85,7 +85,7 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile) erro
return err return err
} }
if len(lane.Validators) > 0 { if len(lane.Validators) > 0 {
return fmt.Errorf("pipeline %q lane %q validators are not supported by the current raw validation runner", id, laneID) return fmt.Errorf("pipeline %q lane %q validators are not supported at artifact lane level; use extract.validators, merge.validators, or normalize.validators", id, laneID)
} }
} }
} }
@@ -108,6 +108,9 @@ func validateBinding(
} }
return fmt.Errorf("pipeline %q %s retries must be greater than or equal to zero", pipelineID, slot) return fmt.Errorf("pipeline %q %s retries must be greater than or equal to zero", pipelineID, slot)
} }
if err := validateValidatorOverride(pipelineID, laneID, slot, binding.Validators); err != nil {
return err
}
if len(binding.References) == 0 { if len(binding.References) == 0 {
return nil return nil
} }
@@ -120,6 +123,36 @@ func validateBinding(
return validateReferenceMapForContext(pipelineID, laneID, slot, binding.References) return validateReferenceMapForContext(pipelineID, laneID, slot, binding.References)
} }
func validateValidatorOverride(pipelineID string, laneID string, slot string, override pipeline.ValidatorOverride) error {
if !override.Set {
return nil
}
switch slot {
case "chunk", "extract", "merge", "normalize":
default:
return fmt.Errorf("%s validators are not supported", referenceContext(pipelineID, laneID, slot))
}
for i, validator := range override.Validators {
context := fmt.Sprintf("%s validators[%d]", referenceContext(pipelineID, laneID, slot), i)
if strings.TrimSpace(validator.Module) == "" {
return fmt.Errorf("%s module must not be empty", context)
}
if len(validator.References) > 0 {
return fmt.Errorf("%s references are not supported", context)
}
if validator.Validators.Set {
return fmt.Errorf("%s nested validators are not supported", context)
}
if validator.Retries != 0 {
return fmt.Errorf("%s retries are not supported", context)
}
if validator.LLMProfile != "" && strings.TrimSpace(validator.LLMProfile) == "" {
return fmt.Errorf("%s llm_profile must not be empty when set", context)
}
}
return nil
}
func validateReferenceMap(pipelineID string, laneID string, references map[string]string) error { func validateReferenceMap(pipelineID string, laneID string, references map[string]string) error {
return validateReferenceMapForContext(pipelineID, laneID, "", references) return validateReferenceMapForContext(pipelineID, laneID, "", references)
} }

View File

@@ -341,13 +341,82 @@ func TestValidateRejectsConfiguredValidators(t *testing.T) {
if err == nil { if err == nil {
t.Fatal("Validate() error = nil, want configured validators error") t.Fatal("Validate() error = nil, want configured validators error")
} }
for _, want := range []string{"example", "events", "validators", "not supported"} { for _, want := range []string{"example", "events", "validators", "extract.validators", "merge.validators", "normalize.validators"} {
if !strings.Contains(err.Error(), want) { if !strings.Contains(err.Error(), want) {
t.Fatalf("Validate() error = %q, want substring %q", err.Error(), want) t.Fatalf("Validate() error = %q, want substring %q", err.Error(), want)
} }
} }
} }
func TestValidateAcceptsStageLocalValidatorOverrides(t *testing.T) {
cfg := validConfig()
profile := cfg.Pipelines["example"]
profile.Chunk.Validators = pipeline.ValidatorOverride{Set: true}
lane := profile.Artifacts["events"]
lane.Extract.Validators = pipeline.ValidatorOverride{
Set: true,
Validators: []pipeline.ModuleBinding{
pipeline.Binding("fake/validator"),
{Module: "fake/llm-validator", LLMProfile: "careful", Options: map[string]any{"threshold": 0.7}},
},
}
lane.Merge.Validators = pipeline.ValidatorOverride{Set: true}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
if err := cfg.Validate(); err != nil {
t.Fatalf("Validate() error = %v, want nil", err)
}
}
func TestValidateRejectsInvalidValidatorBindings(t *testing.T) {
tests := []struct {
name string
binding pipeline.ModuleBinding
want string
}{
{
name: "empty module",
binding: pipeline.ModuleBinding{},
want: "module must not be empty",
},
{
name: "references",
binding: pipeline.ModuleBinding{Module: "fake/validator", References: map[string]string{"roster": "./roster.txt"}},
want: "references are not supported",
},
{
name: "nested validators",
binding: pipeline.ModuleBinding{Module: "fake/validator", Validators: pipeline.ValidatorOverride{Set: true}},
want: "nested validators are not supported",
},
{
name: "retries",
binding: pipeline.ModuleBinding{Module: "fake/validator", Retries: 1},
want: "retries are not supported",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := validConfig()
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Extract.Validators = pipeline.ValidatorOverride{
Set: true,
Validators: []pipeline.ModuleBinding{test.binding},
}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
err := cfg.Validate()
if err == nil || !strings.Contains(err.Error(), test.want) {
t.Fatalf("Validate() error = %v, want %q", err, test.want)
}
})
}
}
func validConfig() Config { func validConfig() Config {
cfg := Default() cfg := Default()
cfg.Pipelines["example"] = pipeline.PipelineProfile{ cfg.Pipelines["example"] = pipeline.PipelineProfile{
@@ -402,6 +471,12 @@ func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.Module
Requires: []string{"normalized"}, Requires: []string{"normalized"},
Provides: []string{"validated"}, Provides: []string{"validated"},
}, },
"fake/llm-validator": {
Key: "fake/llm-validator",
Stage: pipeline.StageValidate,
Requires: []string{"normalized"},
Provides: []string{"validated"},
},
"json": { "json": {
Key: "json", Key: "json",
Stage: pipeline.StageOutput, Stage: pipeline.StageOutput,
@@ -426,6 +501,7 @@ func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.Module
mustRegisterMerger(t, mergers, specs["appendorder"]) mustRegisterMerger(t, mergers, specs["appendorder"])
mustRegisterNormalizer(t, normalizers, specs["noop"]) mustRegisterNormalizer(t, normalizers, specs["noop"])
mustRegisterValidator(t, validators, specs["fake/validator"]) mustRegisterValidator(t, validators, specs["fake/validator"])
mustRegisterValidator(t, validators, specs["fake/llm-validator"])
mustRegisterOutput(t, outputs, specs["json"]) mustRegisterOutput(t, outputs, specs["json"])
return pipeline.ModuleCatalog{ return pipeline.ModuleCatalog{
@@ -477,7 +553,11 @@ func mustRegisterNormalizer(t *testing.T, registry *pipeline.NormalizerRegistry,
func mustRegisterValidator(t *testing.T, registry *pipeline.ValidatorRegistry, spec pipeline.ModuleSpec) { func mustRegisterValidator(t *testing.T, registry *pipeline.ValidatorRegistry, spec pipeline.ModuleSpec) {
t.Helper() t.Helper()
validatorSpec := pipeline.ValidatorSpec{Key: spec.Key, ExecutionClass: contracts.ExecutionClassDeterministic} executionClass := contracts.ExecutionClassDeterministic
if spec.Key == "fake/llm-validator" {
executionClass = contracts.ExecutionClassLLMBacked
}
validatorSpec := pipeline.ValidatorSpec{Key: spec.Key, ExecutionClass: executionClass}
if err := registry.RegisterWithSpec(validatorSpec, func() (contracts.Validator, error) { return nil, nil }); err != nil { if err := registry.RegisterWithSpec(validatorSpec, func() (contracts.Validator, error) { return nil, nil }); err != nil {
t.Fatalf("register validator: %v", err) t.Fatalf("register validator: %v", err)
} }

View File

@@ -25,6 +25,35 @@ type ModuleBinding struct {
Retries int `json:"retries,omitempty"` Retries int `json:"retries,omitempty"`
Options map[string]any `json:"options,omitempty"` Options map[string]any `json:"options,omitempty"`
References map[string]string `json:"references,omitempty"` References map[string]string `json:"references,omitempty"`
Validators ValidatorOverride `json:"validators,omitempty"`
}
type ValidatorOverride struct {
Set bool `json:"set,omitempty"`
Validators []ModuleBinding `json:"validators,omitempty"`
}
func (binding ModuleBinding) MarshalJSON() ([]byte, error) {
type moduleBindingJSON struct {
Module string `json:"module"`
LLMProfile string `json:"llm_profile,omitempty"`
Retries int `json:"retries,omitempty"`
Options map[string]any `json:"options,omitempty"`
References map[string]string `json:"references,omitempty"`
Validators *[]ModuleBinding `json:"validators,omitempty"`
}
out := moduleBindingJSON{
Module: binding.Module,
LLMProfile: binding.LLMProfile,
Retries: binding.Retries,
Options: binding.Options,
References: binding.References,
}
if binding.Validators.Set {
validators := cloneModuleBindings(binding.Validators.Validators)
out.Validators = &validators
}
return json.Marshal(out)
} }
type ArtifactLaneProfile struct { type ArtifactLaneProfile struct {
@@ -186,7 +215,7 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences), ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences),
Output: resolveBinding(profile.Output, DefaultOutputModule), Output: resolveBinding(profile.Output, DefaultOutputModule),
} }
chunkValidatorChain, err := resolveValidatorChain(pipelineID, "", StageChunk, chunk.Module, catalog) chunkValidatorChain, err := resolveValidatorChain(pipelineID, "", StageChunk, chunk.Module, chunk.Validators, catalog)
if err != nil { if err != nil {
return ResolvedPipeline{}, err return ResolvedPipeline{}, err
} }
@@ -316,15 +345,15 @@ func resolveArtifactLane(
return ResolvedArtifactLane{}, nil, nil, configuredValidatorsError(pipelineID, laneID) return ResolvedArtifactLane{}, nil, nil, configuredValidatorsError(pipelineID, laneID)
} }
extractValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageExtract, lane.Extract.Module, catalog) extractValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageExtract, lane.Extract.Module, lane.Extract.Validators, catalog)
if err != nil { if err != nil {
return ResolvedArtifactLane{}, nil, nil, err return ResolvedArtifactLane{}, nil, nil, err
} }
mergeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageMerge, lane.Merge.Module, catalog) mergeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageMerge, lane.Merge.Module, lane.Merge.Validators, catalog)
if err != nil { if err != nil {
return ResolvedArtifactLane{}, nil, nil, err return ResolvedArtifactLane{}, nil, nil, err
} }
normalizeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageNormalize, lane.Normalize.Module, catalog) normalizeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageNormalize, lane.Normalize.Module, lane.Normalize.Validators, catalog)
if err != nil { if err != nil {
return ResolvedArtifactLane{}, nil, nil, err return ResolvedArtifactLane{}, nil, nil, err
} }
@@ -334,10 +363,10 @@ func resolveArtifactLane(
} }
func configuredValidatorsError(pipelineID string, laneID string) error { func configuredValidatorsError(pipelineID string, laneID string) error {
return fmt.Errorf("pipeline %q lane %q configured validators are not supported by the current raw validation runner", pipelineID, laneID) return fmt.Errorf("pipeline %q lane %q validators are not supported at artifact lane level; use extract.validators, merge.validators, or normalize.validators", pipelineID, laneID)
} }
func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage, module string, catalog ModuleCatalog) (ResolvedValidatorChain, error) { func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage, module string, override ValidatorOverride, catalog ModuleCatalog) (ResolvedValidatorChain, error) {
chain := ResolvedValidatorChain{ chain := ResolvedValidatorChain{
Stage: stage, Stage: stage,
LaneID: strings.TrimSpace(laneID), LaneID: strings.TrimSpace(laneID),
@@ -352,10 +381,12 @@ func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage,
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q validator chain stage %q is not supported", pipelineID, stage) return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q validator chain stage %q is not supported", pipelineID, stage)
} }
if catalog.ValidatorChains == nil { var bindings []ModuleBinding
return chain, nil if override.Set {
bindings = cloneModuleBindings(override.Validators)
} else if catalog.ValidatorChains != nil {
bindings = catalog.ValidatorChains.Validators(stage, chain.ModuleKey)
} }
bindings := catalog.ValidatorChains.Validators(stage, chain.ModuleKey)
if len(bindings) == 0 { if len(bindings) == 0 {
return chain, nil return chain, nil
} }
@@ -368,6 +399,9 @@ func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage,
if !ok { if !ok {
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q %s validator chain for module %q references unknown validator %q", pipelineID, stage, chain.ModuleKey, validator.Module) return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q %s validator chain for module %q references unknown validator %q", pipelineID, stage, chain.ModuleKey, validator.Module)
} }
if strings.TrimSpace(validator.LLMProfile) != "" && spec.ExecutionClass != contracts.ExecutionClassLLMBacked {
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q %s validator chain for module %q assigns llm_profile to deterministic validator %q", pipelineID, stage, chain.ModuleKey, validator.Module)
}
chain.Validators = append(chain.Validators, ResolvedValidator{ chain.Validators = append(chain.Validators, ResolvedValidator{
Binding: cloneModuleBinding(validator), Binding: cloneModuleBinding(validator),
ExecutionClass: spec.ExecutionClass, ExecutionClass: spec.ExecutionClass,
@@ -728,6 +762,7 @@ func resolveBinding(binding ModuleBinding, defaultModule string) ModuleBinding {
Retries: binding.Retries, Retries: binding.Retries,
Options: cloneOptions(binding.Options), Options: cloneOptions(binding.Options),
References: normalizeReferenceMap(binding.References), References: normalizeReferenceMap(binding.References),
Validators: cloneValidatorOverride(binding.Validators),
} }
} }

View File

@@ -182,6 +182,143 @@ func TestResolvePipelineRejectsUnknownDefaultValidator(t *testing.T) {
} }
} }
func TestResolvePipelineValidatorOverrideReplacesDefaultChain(t *testing.T) {
catalog := newProfileCatalog(t)
registerProfileValidatorSpec(t, catalog, ValidatorSpec{Key: "second-validator", ExecutionClass: contracts.ExecutionClassLLMBacked})
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
Stage: StageExtract,
Module: "event-extractor",
Validators: []ModuleBinding{Binding("grounded")},
}); err != nil {
t.Fatalf("register validator chain: %v", err)
}
profile := PipelineProfile{
ID: "validated",
Input: Binding("text"),
Artifacts: map[string]ArtifactLaneProfile{
"events": {
Extract: ModuleBinding{
Module: "event-extractor",
Validators: ValidatorOverride{
Set: true,
Validators: []ModuleBinding{
{Module: "second-validator", LLMProfile: "careful"},
Binding("grounded"),
},
},
},
},
},
}
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
extractChain := findResolvedValidatorChain(resolved.ValidatorChains, StageExtract, "events", "event-extractor")
if extractChain == nil {
t.Fatal("extract validator chain not found")
}
if len(extractChain.Validators) != 2 {
t.Fatalf("extract validators = %#v, want explicit two-validator override", extractChain.Validators)
}
if extractChain.Validators[0].Binding.Module != "second-validator" || extractChain.Validators[0].Binding.LLMProfile != "careful" {
t.Fatalf("first validator = %#v, want explicit LLM-backed validator first", extractChain.Validators[0])
}
if extractChain.Validators[1].Binding.Module != "grounded" {
t.Fatalf("second validator = %#v, want grounded second", extractChain.Validators[1])
}
}
func TestResolvePipelineExplicitEmptyValidatorOverrideSuppressesDefaultChain(t *testing.T) {
catalog := newProfileCatalog(t)
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
Stage: StageExtract,
Module: "event-extractor",
Validators: []ModuleBinding{Binding("grounded")},
}); err != nil {
t.Fatalf("register validator chain: %v", err)
}
profile := PipelineProfile{
ID: "validated",
Input: Binding("text"),
Artifacts: map[string]ArtifactLaneProfile{
"events": {
Extract: ModuleBinding{
Module: "event-extractor",
Validators: ValidatorOverride{Set: true},
},
},
},
}
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
extractChain := findResolvedValidatorChain(resolved.ValidatorChains, StageExtract, "events", "event-extractor")
if extractChain == nil {
t.Fatal("extract validator chain not found")
}
if len(extractChain.Validators) != 0 {
t.Fatalf("extract validators = %#v, want explicit empty override", extractChain.Validators)
}
}
func TestResolvePipelineRejectsUnknownOverrideValidator(t *testing.T) {
_, err := ResolvePipeline(PipelineProfile{
ID: "validated",
Input: Binding("text"),
Artifacts: map[string]ArtifactLaneProfile{
"events": {
Extract: ModuleBinding{
Module: "event-extractor",
Validators: ValidatorOverride{
Set: true,
Validators: []ModuleBinding{Binding("missing-validator")},
},
},
},
},
}, ResolveOptions{}, newProfileCatalog(t))
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want unknown validator error")
}
if !strings.Contains(err.Error(), "missing-validator") {
t.Fatalf("ResolvePipeline() error = %q, want missing validator context", err.Error())
}
}
func TestResolvePipelineRejectsLLMProfileForDeterministicValidator(t *testing.T) {
_, err := ResolvePipeline(PipelineProfile{
ID: "validated",
Input: Binding("text"),
Artifacts: map[string]ArtifactLaneProfile{
"events": {
Extract: ModuleBinding{
Module: "event-extractor",
Validators: ValidatorOverride{
Set: true,
Validators: []ModuleBinding{
{Module: "grounded", LLMProfile: "careful"},
},
},
},
},
},
}, ResolveOptions{}, newProfileCatalog(t))
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want deterministic validator profile error")
}
if !strings.Contains(err.Error(), "grounded") || !strings.Contains(err.Error(), "llm_profile") {
t.Fatalf("ResolvePipeline() error = %q, want validator profile context", err.Error())
}
}
func TestResolvePipelineSelectsOnlyRequestedLanes(t *testing.T) { func TestResolvePipelineSelectsOnlyRequestedLanes(t *testing.T) {
profile := multiLaneProfile() profile := multiLaneProfile()
resolved, err := ResolvePipeline(profile, ResolveOptions{Only: []string{" summaries ", "events", "summaries"}}, newProfileCatalog(t)) resolved, err := ResolvePipeline(profile, ResolveOptions{Only: []string{" summaries ", "events", "summaries"}}, newProfileCatalog(t))
@@ -850,7 +987,7 @@ func TestResolvePipelineRejectsConfiguredValidators(t *testing.T) {
if err == nil { if err == nil {
t.Fatal("ResolvePipeline() error = nil, want error") t.Fatal("ResolvePipeline() error = nil, want error")
} }
assertErrorContains(t, err, "baseline", "events", "configured validators", "not supported") assertErrorContains(t, err, "baseline", "events", "validators", "extract.validators")
} }
func TestResolvePipelineOrdersLanesDeterministically(t *testing.T) { func TestResolvePipelineOrdersLanesDeterministically(t *testing.T) {
@@ -1139,6 +1276,13 @@ func registerProfileSpecs(t *testing.T, catalog ModuleCatalog, specs ...ModuleSp
} }
} }
func registerProfileValidatorSpec(t *testing.T, catalog ModuleCatalog, spec ValidatorSpec) {
t.Helper()
if err := catalog.Validators.RegisterWithSpec(spec, profileValidatorConstructor(spec.Key)); err != nil {
t.Fatalf("register validator spec %#v: %v", spec, err)
}
}
func profileInputConstructor(key string) InputAdapterConstructor { func profileInputConstructor(key string) InputAdapterConstructor {
return func() (contracts.InputAdapter, error) { return func() (contracts.InputAdapter, error) {
return profileInputAdapter{key: key}, nil return profileInputAdapter{key: key}, nil

View File

@@ -631,7 +631,7 @@ func validateRunInput(input RunInput) error {
return fmt.Errorf("resolved pipeline lane %q normalize module must not be empty", lane.ID) return fmt.Errorf("resolved pipeline lane %q normalize module must not be empty", lane.ID)
} }
if len(lane.Validators) > 0 { if len(lane.Validators) > 0 {
return fmt.Errorf("resolved pipeline lane %q configured validators are not supported by the current raw validation runner", lane.ID) return fmt.Errorf("resolved pipeline lane %q validators are not supported at artifact lane level; use extract.validators, merge.validators, or normalize.validators", lane.ID)
} }
} }
return nil return nil

View File

@@ -1081,7 +1081,7 @@ func TestRunRejectsConfiguredValidators(t *testing.T) {
_, err := New(newRunnerRegistries(t, defaultRunnerModules())).Run(context.Background(), RunInput{ _, err := New(newRunnerRegistries(t, defaultRunnerModules())).Run(context.Background(), RunInput{
Pipeline: resolvedPipelineWithValidators("configured", "second-validator"), Pipeline: resolvedPipelineWithValidators("configured", "second-validator"),
}) })
assertRunError(t, err, "configured validators") assertRunError(t, err, "extract.validators")
} }
func TestRunCollectsStageWarnings(t *testing.T) { func TestRunCollectsStageWarnings(t *testing.T) {

View File

@@ -98,5 +98,13 @@ func cloneModuleBinding(binding ModuleBinding) ModuleBinding {
} }
binding.References = references binding.References = references
} }
binding.Validators = cloneValidatorOverride(binding.Validators)
return binding return binding
} }
func cloneValidatorOverride(override ValidatorOverride) ValidatorOverride {
return ValidatorOverride{
Set: override.Set,
Validators: cloneModuleBindings(override.Validators),
}
}