eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add validator chain config overrides

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-07 21:21:12 +00:00
parent d593bfee0a
commit 666b4bf801
17 changed files with 492 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
docs/cli.md
View File

@@ -45,8 +45,9 @@ Flags:
Defaults to `./notarius-output`.
- `--diagnostics-dir path`: diagnostics work directory override for this
invocation.
- `--llm-profile id`: override every effective LLM-capable module binding to
use one Scriptorium profile ID.
- `--llm-profile id`: override every effective LLM-capable pipeline module
binding to use one Scriptorium profile ID. Validator-specific profiles are
not overridden.
- `--session-id id`: pass a stable prompt session identifier through LLM-backed
module calls.
- `--reference selector=path`: bind a reference path to a chunk, extractor,

27
docs/config.md
View File

@@ -131,8 +131,9 @@ Artifact lane fields:
- `extract`: required module binding.
- `merge`: optional module binding. Default module is `appendorder`.
- `normalize`: optional module binding. Default module is `noop`.
- `validators`: reserved for future configurable validator chains. Non-empty
lists are rejected by current configuration validation.
- `validators`: deprecated lane-level validator list. Non-empty lists are
rejected; use `extract.validators`, `merge.validators`, or
`normalize.validators`.
- `references`: optional compatibility alias for extractor reference bindings.
Lane bindings override pipeline-level bindings for the same slot.
@@ -234,12 +235,28 @@ Binding fields:
- `options`: optional module-specific settings.
- `references`: optional reference bindings. Supported only for `chunk`,
`extract`, `merge`, and `normalize` bindings. `input` and `output` bindings
reject this field during validation. Validator bindings are reserved for a
future validator-chain feature and are rejected when configured.
reject this field during validation.
- `validators`: optional stage-local validator chain override. Supported only
for `chunk`, `extract`, `merge`, and `normalize` bindings. Omit the field to
use the production default chain; set `validators: []` to force an empty
chain; set a non-empty list to use exactly those validators in configured
order.
Validator bindings use the same shorthand or object module-binding form, but
only these fields are supported:
- `module`: validator key.
- `llm_profile`: optional Scriptorium profile ID for LLM-backed validators.
- `options`: optional validator-specific settings.
Validator bindings reject `references`, `retries`, and nested `validators`.
During resolution, deterministic validators reject explicit `llm_profile`
values.
The `--llm-profile` run flag overrides every effective LLM-capable module
binding to use one Scriptorium profile ID: chunk, every selected lane extract,
merge, and normalize binding.
merge, and normalize binding. It does not override validator-specific
`llm_profile` values.
## Implemented Production Modules

8
docs/internal/pipeline.md
View File

@@ -187,10 +187,10 @@ Runner-side raw validation chains receive the raw module output plus
stage, lane, module, source, and chunk provenance. Empty raw validation chains
approve output by default.
Pipeline-configured validator lists are not part of the current runner
contract. Non-empty configured validator lists are rejected during configuration
validation or resolved-run validation so they cannot appear in manifests without
executing.
Resolved validator chains come from central default mappings unless a
stage-local config override is set on `chunk`, lane `extract`, lane `merge`, or
lane `normalize`. Explicit empty overrides are valid and are recorded as empty
chains in manifests.
Validator rejection is a non-fatal run outcome: the rejected output is recorded
in `RunOutput.Rejected` and does not pass to the next stage. Validator execution

7
internal/cli/run.go
View File

@@ -495,6 +495,13 @@ func effectiveLLMProfileIDs(resolved pipeline.ResolvedPipeline) []string {
add(lane.Merge)
add(lane.Normalize)
}
for _, chain := range resolved.ValidatorChains {
for _, validator := range chain.Validators {
if validator.ExecutionClass == contracts.ExecutionClassLLMBacked {
add(validator.Binding)
}
}
}
ids := make([]string, 0, len(seen))
for id := range seen {
ids = append(ids, id)

26
internal/cli/run_test.go
View File

@@ -858,16 +858,32 @@ func TestEffectiveLLMProfileIDsUsesLLMCapableStagesOnly(t *testing.T) {
Output: pipeline.ModuleBinding{LLMProfile: "output-profile"},
ArtifactLanes: []pipeline.ResolvedArtifactLane{
{
Extract: pipeline.ModuleBinding{LLMProfile: "extract-profile"},
Merge: pipeline.ModuleBinding{LLMProfile: "merge-profile"},
Normalize: pipeline.ModuleBinding{LLMProfile: "normalize-profile"},
Validators: []pipeline.ModuleBinding{{LLMProfile: "validator-profile"}},
Extract: pipeline.ModuleBinding{LLMProfile: "extract-profile"},
Merge: pipeline.ModuleBinding{LLMProfile: "merge-profile"},
Normalize: pipeline.ModuleBinding{LLMProfile: "normalize-profile"},
},
},
ValidatorChains: []pipeline.ResolvedValidatorChain{
{
Stage: pipeline.StageExtract,
LaneID: "events",
ModuleKey: "extract",
Validators: []pipeline.ResolvedValidator{
{
Binding: pipeline.ModuleBinding{Module: "deterministic-validator", LLMProfile: "ignored-validator-profile"},
ExecutionClass: contracts.ExecutionClassDeterministic,
},
{
Binding: pipeline.ModuleBinding{Module: "llm-validator", LLMProfile: "validator-profile"},
ExecutionClass: contracts.ExecutionClassLLMBacked,
},
},
},
},
}
got := effectiveLLMProfileIDs(resolved)
want := []string{"chunk-profile", "extract-profile", "merge-profile", "normalize-profile"}
want := []string{"chunk-profile", "extract-profile", "merge-profile", "normalize-profile", "validator-profile"}
if !reflect.DeepEqual(got, want) {
t.Fatalf("effectiveLLMProfileIDs() = %#v, want %#v", got, want)
}

12
internal/core/config/config.go
View File

@@ -97,6 +97,18 @@ func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
out.Options = cloneOptions(in.Options)
}
out.References = cloneStringMap(in.References)
out.Validators = cloneValidatorOverride(in.Validators)
return out
}
func cloneValidatorOverride(in pipeline.ValidatorOverride) pipeline.ValidatorOverride {
out := pipeline.ValidatorOverride{Set: in.Set}
if len(in.Validators) > 0 {
out.Validators = make([]pipeline.ModuleBinding, len(in.Validators))
for i, binding := range in.Validators {
out.Validators[i] = cloneModuleBinding(binding)
}
}
return out
}

23
internal/core/config/effective_config_test.go
View File

@@ -161,6 +161,12 @@ func TestResolveLLMProfileOverrideAppliesBeforeDigest(t *testing.T) {
profile.Output.LLMProfile = "output-profile"
lane := profile.Artifacts["events"]
lane.Merge.LLMProfile = "merge-profile"
lane.Extract.Validators = pipeline.ValidatorOverride{
Set: true,
Validators: []pipeline.ModuleBinding{
{Module: "fake/llm-validator", LLMProfile: "validator-profile"},
},
}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
@@ -195,9 +201,22 @@ func TestResolveLLMProfileOverrideAppliesBeforeDigest(t *testing.T) {
if eventLane.Merge.LLMProfile != "runtime" {
t.Fatalf("merge profile = %q, want runtime", eventLane.Merge.LLMProfile)
}
if len(eventLane.Validators) != 0 {
t.Fatalf("validator profiles = %#v, want none", eventLane.Validators)
validatorChain := findEffectiveValidatorChain(effective.ResolvedPipeline.ValidatorChains, pipeline.StageExtract, "events", "fake/extract")
if validatorChain == nil || len(validatorChain.Validators) != 1 {
t.Fatalf("validator chain = %#v, want one extract validator", effective.ResolvedPipeline.ValidatorChains)
}
if validatorChain.Validators[0].Binding.LLMProfile != "validator-profile" {
t.Fatalf("validator profile = %q, want original validator-profile", validatorChain.Validators[0].Binding.LLMProfile)
}
}
func findEffectiveValidatorChain(chains []pipeline.ResolvedValidatorChain, stage pipeline.ModuleStage, laneID string, module string) *pipeline.ResolvedValidatorChain {
for i := range chains {
if chains[i].Stage == stage && chains[i].LaneID == laneID && chains[i].ModuleKey == module {
return &chains[i]
}
}
return nil
}
func llmCapableBindings(resolved pipeline.ResolvedPipeline) []pipeline.ModuleBinding {

12
internal/core/config/file_config.go
View File

@@ -56,6 +56,7 @@ type fileModuleBinding struct {
Retries int
Options map[string]any
References map[string]string
Validators pipeline.ValidatorOverride
}
func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
@@ -102,6 +103,16 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
return err
}
b.References = references
case "validators":
b.Validators.Set = true
var validators []fileModuleBinding
if err := valueNode.Decode(&validators); err != nil {
return err
}
b.Validators.Validators = make([]pipeline.ModuleBinding, len(validators))
for i, validator := range validators {
b.Validators.Validators[i] = validator.toPipelineBinding()
}
default:
return fmt.Errorf("field %s not found in module binding", keyNode.Value)
}
@@ -119,6 +130,7 @@ func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
Retries: b.Retries,
Options: cloneOptions(b.Options),
References: normalizedStringMap(b.References),
Validators: b.Validators,
}
}

55
internal/core/config/file_config_test.go
View File

@@ -300,6 +300,61 @@ pipelines:
}
}
func TestParseFileConfigStageLocalValidatorOverrides(t *testing.T) {
cfg := parseAndApplyConfig(t, `
version: 2
pipelines:
example:
input: fake/input
chunk:
module: generic
validators: []
artifacts:
events:
extract:
module: fake/extract
validators:
- fake/validator
- module: fake/llm-validator
llm_profile: careful
options:
threshold: 0.7
merge:
module: appendorder
validators: []
normalize:
module: noop
`)
profile := cfg.Pipelines["example"]
if !profile.Chunk.Validators.Set || len(profile.Chunk.Validators.Validators) != 0 {
t.Fatalf("chunk validator override = %#v, want explicit empty", profile.Chunk.Validators)
}
lane := profile.Artifacts["events"]
if !lane.Extract.Validators.Set {
t.Fatalf("extract validator override Set = false, want true")
}
validators := lane.Extract.Validators.Validators
if len(validators) != 2 {
t.Fatalf("extract validators = %#v, want two validators", validators)
}
if validators[0].Module != "fake/validator" {
t.Fatalf("first validator = %#v, want fake/validator", validators[0])
}
if validators[1].Module != "fake/llm-validator" || validators[1].LLMProfile != "careful" {
t.Fatalf("second validator = %#v, want LLM validator with profile", validators[1])
}
if validators[1].Options["threshold"] != 0.7 {
t.Fatalf("second validator options = %#v, want threshold", validators[1].Options)
}
if !lane.Merge.Validators.Set || len(lane.Merge.Validators.Validators) != 0 {
t.Fatalf("merge validator override = %#v, want explicit empty", lane.Merge.Validators)
}
if lane.Normalize.Validators.Set {
t.Fatalf("normalize validator override Set = true, want omitted")
}
}
func TestApplyFileConfigRejectsDuplicateTrimmedPipelineIDs(t *testing.T) {
fileCfg, err := ParseFileConfigYAML([]byte(`
version: 2

20
internal/core/config/redaction.go
View File

@@ -27,6 +27,12 @@ func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeli
out.Chunk = cloneModuleBinding(in.Chunk)
out.ChunkReferences = pipeline.CloneReferenceTarget(in.ChunkReferences)
out.Output = cloneModuleBinding(in.Output)
if len(in.ValidatorChains) > 0 {
out.ValidatorChains = make([]pipeline.ResolvedValidatorChain, len(in.ValidatorChains))
for i, chain := range in.ValidatorChains {
out.ValidatorChains[i] = cloneResolvedValidatorChain(chain)
}
}
if len(in.ArtifactLanes) > 0 {
out.ArtifactLanes = make([]pipeline.ResolvedArtifactLane, len(in.ArtifactLanes))
for i, lane := range in.ArtifactLanes {
@@ -36,6 +42,20 @@ func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeli
return out
}
func cloneResolvedValidatorChain(in pipeline.ResolvedValidatorChain) pipeline.ResolvedValidatorChain {
out := in
if len(in.Validators) > 0 {
out.Validators = make([]pipeline.ResolvedValidator, len(in.Validators))
for i, validator := range in.Validators {
out.Validators[i] = pipeline.ResolvedValidator{
Binding: cloneModuleBinding(validator.Binding),
ExecutionClass: validator.ExecutionClass,
}
}
}
return out
}
func cloneResolvedArtifactLane(in pipeline.ResolvedArtifactLane) pipeline.ResolvedArtifactLane {
out := in
out.Extract = cloneModuleBinding(in.Extract)

35
internal/core/config/validation.go
View File

@@ -85,7 +85,7 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile) erro
return err
}
if len(lane.Validators) > 0 {
return fmt.Errorf("pipeline %q lane %q validators are not supported by the current raw validation runner", id, laneID)
return fmt.Errorf("pipeline %q lane %q validators are not supported at artifact lane level; use extract.validators, merge.validators, or normalize.validators", id, laneID)
}
}
}
@@ -108,6 +108,9 @@ func validateBinding(
}
return fmt.Errorf("pipeline %q %s retries must be greater than or equal to zero", pipelineID, slot)
}
if err := validateValidatorOverride(pipelineID, laneID, slot, binding.Validators); err != nil {
return err
}
if len(binding.References) == 0 {
return nil
}
@@ -120,6 +123,36 @@ func validateBinding(
return validateReferenceMapForContext(pipelineID, laneID, slot, binding.References)
}
func validateValidatorOverride(pipelineID string, laneID string, slot string, override pipeline.ValidatorOverride) error {
if !override.Set {
return nil
}
switch slot {
case "chunk", "extract", "merge", "normalize":
default:
return fmt.Errorf("%s validators are not supported", referenceContext(pipelineID, laneID, slot))
}
for i, validator := range override.Validators {
context := fmt.Sprintf("%s validators[%d]", referenceContext(pipelineID, laneID, slot), i)
if strings.TrimSpace(validator.Module) == "" {
return fmt.Errorf("%s module must not be empty", context)
}
if len(validator.References) > 0 {
return fmt.Errorf("%s references are not supported", context)
}
if validator.Validators.Set {
return fmt.Errorf("%s nested validators are not supported", context)
}
if validator.Retries != 0 {
return fmt.Errorf("%s retries are not supported", context)
}
if validator.LLMProfile != "" && strings.TrimSpace(validator.LLMProfile) == "" {
return fmt.Errorf("%s llm_profile must not be empty when set", context)
}
}
return nil
}
func validateReferenceMap(pipelineID string, laneID string, references map[string]string) error {
return validateReferenceMapForContext(pipelineID, laneID, "", references)
}

84
internal/core/config/validation_test.go
View File

@@ -341,13 +341,82 @@ func TestValidateRejectsConfiguredValidators(t *testing.T) {
if err == nil {
t.Fatal("Validate() error = nil, want configured validators error")
}
for _, want := range []string{"example", "events", "validators", "not supported"} {
for _, want := range []string{"example", "events", "validators", "extract.validators", "merge.validators", "normalize.validators"} {
if !strings.Contains(err.Error(), want) {
t.Fatalf("Validate() error = %q, want substring %q", err.Error(), want)
}
}
}
func TestValidateAcceptsStageLocalValidatorOverrides(t *testing.T) {
cfg := validConfig()
profile := cfg.Pipelines["example"]
profile.Chunk.Validators = pipeline.ValidatorOverride{Set: true}
lane := profile.Artifacts["events"]
lane.Extract.Validators = pipeline.ValidatorOverride{
Set: true,
Validators: []pipeline.ModuleBinding{
pipeline.Binding("fake/validator"),
{Module: "fake/llm-validator", LLMProfile: "careful", Options: map[string]any{"threshold": 0.7}},
},
}
lane.Merge.Validators = pipeline.ValidatorOverride{Set: true}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
if err := cfg.Validate(); err != nil {
t.Fatalf("Validate() error = %v, want nil", err)
}
}
func TestValidateRejectsInvalidValidatorBindings(t *testing.T) {
tests := []struct {
name string
binding pipeline.ModuleBinding
want string
}{
{
name: "empty module",
binding: pipeline.ModuleBinding{},
want: "module must not be empty",
},
{
name: "references",
binding: pipeline.ModuleBinding{Module: "fake/validator", References: map[string]string{"roster": "./roster.txt"}},
want: "references are not supported",
},
{
name: "nested validators",
binding: pipeline.ModuleBinding{Module: "fake/validator", Validators: pipeline.ValidatorOverride{Set: true}},
want: "nested validators are not supported",
},
{
name: "retries",
binding: pipeline.ModuleBinding{Module: "fake/validator", Retries: 1},
want: "retries are not supported",
},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
cfg := validConfig()
profile := cfg.Pipelines["example"]
lane := profile.Artifacts["events"]
lane.Extract.Validators = pipeline.ValidatorOverride{
Set: true,
Validators: []pipeline.ModuleBinding{test.binding},
}
profile.Artifacts["events"] = lane
cfg.Pipelines["example"] = profile
err := cfg.Validate()
if err == nil || !strings.Contains(err.Error(), test.want) {
t.Fatalf("Validate() error = %v, want %q", err, test.want)
}
})
}
}
func validConfig() Config {
cfg := Default()
cfg.Pipelines["example"] = pipeline.PipelineProfile{
@@ -402,6 +471,12 @@ func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.Module
Requires: []string{"normalized"},
Provides: []string{"validated"},
},
"fake/llm-validator": {
Key: "fake/llm-validator",
Stage: pipeline.StageValidate,
Requires: []string{"normalized"},
Provides: []string{"validated"},
},
"json": {
Key: "json",
Stage: pipeline.StageOutput,
@@ -426,6 +501,7 @@ func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.Module
mustRegisterMerger(t, mergers, specs["appendorder"])
mustRegisterNormalizer(t, normalizers, specs["noop"])
mustRegisterValidator(t, validators, specs["fake/validator"])
mustRegisterValidator(t, validators, specs["fake/llm-validator"])
mustRegisterOutput(t, outputs, specs["json"])
return pipeline.ModuleCatalog{
@@ -477,7 +553,11 @@ func mustRegisterNormalizer(t *testing.T, registry *pipeline.NormalizerRegistry,
func mustRegisterValidator(t *testing.T, registry *pipeline.ValidatorRegistry, spec pipeline.ModuleSpec) {
t.Helper()
validatorSpec := pipeline.ValidatorSpec{Key: spec.Key, ExecutionClass: contracts.ExecutionClassDeterministic}
executionClass := contracts.ExecutionClassDeterministic
if spec.Key == "fake/llm-validator" {
executionClass = contracts.ExecutionClassLLMBacked
}
validatorSpec := pipeline.ValidatorSpec{Key: spec.Key, ExecutionClass: executionClass}
if err := registry.RegisterWithSpec(validatorSpec, func() (contracts.Validator, error) { return nil, nil }); err != nil {
t.Fatalf("register validator: %v", err)
}

53
internal/framework/pipeline/profile.go
View File

@@ -25,6 +25,35 @@ type ModuleBinding struct {
Retries int `json:"retries,omitempty"`
Options map[string]any `json:"options,omitempty"`
References map[string]string `json:"references,omitempty"`
Validators ValidatorOverride `json:"validators,omitempty"`
}
type ValidatorOverride struct {
Set bool `json:"set,omitempty"`
Validators []ModuleBinding `json:"validators,omitempty"`
}
func (binding ModuleBinding) MarshalJSON() ([]byte, error) {
type moduleBindingJSON struct {
Module string `json:"module"`
LLMProfile string `json:"llm_profile,omitempty"`
Retries int `json:"retries,omitempty"`
Options map[string]any `json:"options,omitempty"`
References map[string]string `json:"references,omitempty"`
Validators *[]ModuleBinding `json:"validators,omitempty"`
}
out := moduleBindingJSON{
Module: binding.Module,
LLMProfile: binding.LLMProfile,
Retries: binding.Retries,
Options: binding.Options,
References: binding.References,
}
if binding.Validators.Set {
validators := cloneModuleBindings(binding.Validators.Validators)
out.Validators = &validators
}
return json.Marshal(out)
}
type ArtifactLaneProfile struct {
@@ -186,7 +215,7 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences),
Output: resolveBinding(profile.Output, DefaultOutputModule),
}
chunkValidatorChain, err := resolveValidatorChain(pipelineID, "", StageChunk, chunk.Module, catalog)
chunkValidatorChain, err := resolveValidatorChain(pipelineID, "", StageChunk, chunk.Module, chunk.Validators, catalog)
if err != nil {
return ResolvedPipeline{}, err
}
@@ -316,15 +345,15 @@ func resolveArtifactLane(
return ResolvedArtifactLane{}, nil, nil, configuredValidatorsError(pipelineID, laneID)
}
extractValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageExtract, lane.Extract.Module, catalog)
extractValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageExtract, lane.Extract.Module, lane.Extract.Validators, catalog)
if err != nil {
return ResolvedArtifactLane{}, nil, nil, err
}
mergeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageMerge, lane.Merge.Module, catalog)
mergeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageMerge, lane.Merge.Module, lane.Merge.Validators, catalog)
if err != nil {
return ResolvedArtifactLane{}, nil, nil, err
}
normalizeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageNormalize, lane.Normalize.Module, catalog)
normalizeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageNormalize, lane.Normalize.Module, lane.Normalize.Validators, catalog)
if err != nil {
return ResolvedArtifactLane{}, nil, nil, err
}
@@ -334,10 +363,10 @@ func resolveArtifactLane(
}
func configuredValidatorsError(pipelineID string, laneID string) error {
return fmt.Errorf("pipeline %q lane %q configured validators are not supported by the current raw validation runner", pipelineID, laneID)
return fmt.Errorf("pipeline %q lane %q validators are not supported at artifact lane level; use extract.validators, merge.validators, or normalize.validators", pipelineID, laneID)
}
func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage, module string, catalog ModuleCatalog) (ResolvedValidatorChain, error) {
func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage, module string, override ValidatorOverride, catalog ModuleCatalog) (ResolvedValidatorChain, error) {
chain := ResolvedValidatorChain{
Stage: stage,
LaneID: strings.TrimSpace(laneID),
@@ -352,10 +381,12 @@ func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage,
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q validator chain stage %q is not supported", pipelineID, stage)
}
if catalog.ValidatorChains == nil {
return chain, nil
var bindings []ModuleBinding
if override.Set {
bindings = cloneModuleBindings(override.Validators)
} else if catalog.ValidatorChains != nil {
bindings = catalog.ValidatorChains.Validators(stage, chain.ModuleKey)
}
bindings := catalog.ValidatorChains.Validators(stage, chain.ModuleKey)
if len(bindings) == 0 {
return chain, nil
}
@@ -368,6 +399,9 @@ func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage,
if !ok {
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q %s validator chain for module %q references unknown validator %q", pipelineID, stage, chain.ModuleKey, validator.Module)
}
if strings.TrimSpace(validator.LLMProfile) != "" && spec.ExecutionClass != contracts.ExecutionClassLLMBacked {
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q %s validator chain for module %q assigns llm_profile to deterministic validator %q", pipelineID, stage, chain.ModuleKey, validator.Module)
}
chain.Validators = append(chain.Validators, ResolvedValidator{
Binding: cloneModuleBinding(validator),
ExecutionClass: spec.ExecutionClass,
@@ -728,6 +762,7 @@ func resolveBinding(binding ModuleBinding, defaultModule string) ModuleBinding {
Retries: binding.Retries,
Options: cloneOptions(binding.Options),
References: normalizeReferenceMap(binding.References),
Validators: cloneValidatorOverride(binding.Validators),
}
}

146
internal/framework/pipeline/profile_test.go
View File

@@ -182,6 +182,143 @@ func TestResolvePipelineRejectsUnknownDefaultValidator(t *testing.T) {
}
}
func TestResolvePipelineValidatorOverrideReplacesDefaultChain(t *testing.T) {
catalog := newProfileCatalog(t)
registerProfileValidatorSpec(t, catalog, ValidatorSpec{Key: "second-validator", ExecutionClass: contracts.ExecutionClassLLMBacked})
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
Stage: StageExtract,
Module: "event-extractor",
Validators: []ModuleBinding{Binding("grounded")},
}); err != nil {
t.Fatalf("register validator chain: %v", err)
}
profile := PipelineProfile{
ID: "validated",
Input: Binding("text"),
Artifacts: map[string]ArtifactLaneProfile{
"events": {
Extract: ModuleBinding{
Module: "event-extractor",
Validators: ValidatorOverride{
Set: true,
Validators: []ModuleBinding{
{Module: "second-validator", LLMProfile: "careful"},
Binding("grounded"),
},
},
},
},
},
}
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
extractChain := findResolvedValidatorChain(resolved.ValidatorChains, StageExtract, "events", "event-extractor")
if extractChain == nil {
t.Fatal("extract validator chain not found")
}
if len(extractChain.Validators) != 2 {
t.Fatalf("extract validators = %#v, want explicit two-validator override", extractChain.Validators)
}
if extractChain.Validators[0].Binding.Module != "second-validator" || extractChain.Validators[0].Binding.LLMProfile != "careful" {
t.Fatalf("first validator = %#v, want explicit LLM-backed validator first", extractChain.Validators[0])
}
if extractChain.Validators[1].Binding.Module != "grounded" {
t.Fatalf("second validator = %#v, want grounded second", extractChain.Validators[1])
}
}
func TestResolvePipelineExplicitEmptyValidatorOverrideSuppressesDefaultChain(t *testing.T) {
catalog := newProfileCatalog(t)
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
Stage: StageExtract,
Module: "event-extractor",
Validators: []ModuleBinding{Binding("grounded")},
}); err != nil {
t.Fatalf("register validator chain: %v", err)
}
profile := PipelineProfile{
ID: "validated",
Input: Binding("text"),
Artifacts: map[string]ArtifactLaneProfile{
"events": {
Extract: ModuleBinding{
Module: "event-extractor",
Validators: ValidatorOverride{Set: true},
},
},
},
}
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
extractChain := findResolvedValidatorChain(resolved.ValidatorChains, StageExtract, "events", "event-extractor")
if extractChain == nil {
t.Fatal("extract validator chain not found")
}
if len(extractChain.Validators) != 0 {
t.Fatalf("extract validators = %#v, want explicit empty override", extractChain.Validators)
}
}
func TestResolvePipelineRejectsUnknownOverrideValidator(t *testing.T) {
_, err := ResolvePipeline(PipelineProfile{
ID: "validated",
Input: Binding("text"),
Artifacts: map[string]ArtifactLaneProfile{
"events": {
Extract: ModuleBinding{
Module: "event-extractor",
Validators: ValidatorOverride{
Set: true,
Validators: []ModuleBinding{Binding("missing-validator")},
},
},
},
},
}, ResolveOptions{}, newProfileCatalog(t))
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want unknown validator error")
}
if !strings.Contains(err.Error(), "missing-validator") {
t.Fatalf("ResolvePipeline() error = %q, want missing validator context", err.Error())
}
}
func TestResolvePipelineRejectsLLMProfileForDeterministicValidator(t *testing.T) {
_, err := ResolvePipeline(PipelineProfile{
ID: "validated",
Input: Binding("text"),
Artifacts: map[string]ArtifactLaneProfile{
"events": {
Extract: ModuleBinding{
Module: "event-extractor",
Validators: ValidatorOverride{
Set: true,
Validators: []ModuleBinding{
{Module: "grounded", LLMProfile: "careful"},
},
},
},
},
},
}, ResolveOptions{}, newProfileCatalog(t))
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want deterministic validator profile error")
}
if !strings.Contains(err.Error(), "grounded") || !strings.Contains(err.Error(), "llm_profile") {
t.Fatalf("ResolvePipeline() error = %q, want validator profile context", err.Error())
}
}
func TestResolvePipelineSelectsOnlyRequestedLanes(t *testing.T) {
profile := multiLaneProfile()
resolved, err := ResolvePipeline(profile, ResolveOptions{Only: []string{" summaries ", "events", "summaries"}}, newProfileCatalog(t))
@@ -850,7 +987,7 @@ func TestResolvePipelineRejectsConfiguredValidators(t *testing.T) {
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want error")
}
assertErrorContains(t, err, "baseline", "events", "configured validators", "not supported")
assertErrorContains(t, err, "baseline", "events", "validators", "extract.validators")
}
func TestResolvePipelineOrdersLanesDeterministically(t *testing.T) {
@@ -1139,6 +1276,13 @@ func registerProfileSpecs(t *testing.T, catalog ModuleCatalog, specs ...ModuleSp
}
}
func registerProfileValidatorSpec(t *testing.T, catalog ModuleCatalog, spec ValidatorSpec) {
t.Helper()
if err := catalog.Validators.RegisterWithSpec(spec, profileValidatorConstructor(spec.Key)); err != nil {
t.Fatalf("register validator spec %#v: %v", spec, err)
}
}
func profileInputConstructor(key string) InputAdapterConstructor {
return func() (contracts.InputAdapter, error) {
return profileInputAdapter{key: key}, nil

2
internal/framework/pipeline/runner.go
View File

@@ -631,7 +631,7 @@ func validateRunInput(input RunInput) error {
return fmt.Errorf("resolved pipeline lane %q normalize module must not be empty", lane.ID)
}
if len(lane.Validators) > 0 {
return fmt.Errorf("resolved pipeline lane %q configured validators are not supported by the current raw validation runner", lane.ID)
return fmt.Errorf("resolved pipeline lane %q validators are not supported at artifact lane level; use extract.validators, merge.validators, or normalize.validators", lane.ID)
}
}
return nil

2
internal/framework/pipeline/runner_test.go
View File

@@ -1081,7 +1081,7 @@ func TestRunRejectsConfiguredValidators(t *testing.T) {
_, err := New(newRunnerRegistries(t, defaultRunnerModules())).Run(context.Background(), RunInput{
Pipeline: resolvedPipelineWithValidators("configured", "second-validator"),
})
assertRunError(t, err, "configured validators")
assertRunError(t, err, "extract.validators")
}
func TestRunCollectsStageWarnings(t *testing.T) {

8
internal/framework/pipeline/validator_chain_registry.go
View File

@@ -98,5 +98,13 @@ func cloneModuleBinding(binding ModuleBinding) ModuleBinding {
}
binding.References = references
}
binding.Validators = cloneValidatorOverride(binding.Validators)
return binding
}
func cloneValidatorOverride(override ValidatorOverride) ValidatorOverride {
return ValidatorOverride{
Set: override.Set,
Validators: cloneModuleBindings(override.Validators),
}
}