eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add validator chain config overrides

Browse Source
This commit is contained in:
Eric Rakestraw
2026-07-07 21:21:12 +00:00
parent d593bfee0a
commit 666b4bf801
17 changed files with 492 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

53
internal/framework/pipeline/profile.go
View File

@@ -25,6 +25,35 @@ type ModuleBinding struct {
Retries int `json:"retries,omitempty"`
Options map[string]any `json:"options,omitempty"`
References map[string]string `json:"references,omitempty"`
Validators ValidatorOverride `json:"validators,omitempty"`
}
type ValidatorOverride struct {
Set bool `json:"set,omitempty"`
Validators []ModuleBinding `json:"validators,omitempty"`
}
func (binding ModuleBinding) MarshalJSON() ([]byte, error) {
type moduleBindingJSON struct {
Module string `json:"module"`
LLMProfile string `json:"llm_profile,omitempty"`
Retries int `json:"retries,omitempty"`
Options map[string]any `json:"options,omitempty"`
References map[string]string `json:"references,omitempty"`
Validators *[]ModuleBinding `json:"validators,omitempty"`
}
out := moduleBindingJSON{
Module: binding.Module,
LLMProfile: binding.LLMProfile,
Retries: binding.Retries,
Options: binding.Options,
References: binding.References,
}
if binding.Validators.Set {
validators := cloneModuleBindings(binding.Validators.Validators)
out.Validators = &validators
}
return json.Marshal(out)
}
type ArtifactLaneProfile struct {
@@ -186,7 +215,7 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences),
Output: resolveBinding(profile.Output, DefaultOutputModule),
}
chunkValidatorChain, err := resolveValidatorChain(pipelineID, "", StageChunk, chunk.Module, catalog)
chunkValidatorChain, err := resolveValidatorChain(pipelineID, "", StageChunk, chunk.Module, chunk.Validators, catalog)
if err != nil {
return ResolvedPipeline{}, err
}
@@ -316,15 +345,15 @@ func resolveArtifactLane(
return ResolvedArtifactLane{}, nil, nil, configuredValidatorsError(pipelineID, laneID)
}
extractValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageExtract, lane.Extract.Module, catalog)
extractValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageExtract, lane.Extract.Module, lane.Extract.Validators, catalog)
if err != nil {
return ResolvedArtifactLane{}, nil, nil, err
}
mergeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageMerge, lane.Merge.Module, catalog)
mergeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageMerge, lane.Merge.Module, lane.Merge.Validators, catalog)
if err != nil {
return ResolvedArtifactLane{}, nil, nil, err
}
normalizeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageNormalize, lane.Normalize.Module, catalog)
normalizeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageNormalize, lane.Normalize.Module, lane.Normalize.Validators, catalog)
if err != nil {
return ResolvedArtifactLane{}, nil, nil, err
}
@@ -334,10 +363,10 @@ func resolveArtifactLane(
}
func configuredValidatorsError(pipelineID string, laneID string) error {
return fmt.Errorf("pipeline %q lane %q configured validators are not supported by the current raw validation runner", pipelineID, laneID)
return fmt.Errorf("pipeline %q lane %q validators are not supported at artifact lane level; use extract.validators, merge.validators, or normalize.validators", pipelineID, laneID)
}
func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage, module string, catalog ModuleCatalog) (ResolvedValidatorChain, error) {
func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage, module string, override ValidatorOverride, catalog ModuleCatalog) (ResolvedValidatorChain, error) {
chain := ResolvedValidatorChain{
Stage: stage,
LaneID: strings.TrimSpace(laneID),
@@ -352,10 +381,12 @@ func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage,
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q validator chain stage %q is not supported", pipelineID, stage)
}
if catalog.ValidatorChains == nil {
return chain, nil
var bindings []ModuleBinding
if override.Set {
bindings = cloneModuleBindings(override.Validators)
} else if catalog.ValidatorChains != nil {
bindings = catalog.ValidatorChains.Validators(stage, chain.ModuleKey)
}
bindings := catalog.ValidatorChains.Validators(stage, chain.ModuleKey)
if len(bindings) == 0 {
return chain, nil
}
@@ -368,6 +399,9 @@ func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage,
if !ok {
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q %s validator chain for module %q references unknown validator %q", pipelineID, stage, chain.ModuleKey, validator.Module)
}
if strings.TrimSpace(validator.LLMProfile) != "" && spec.ExecutionClass != contracts.ExecutionClassLLMBacked {
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q %s validator chain for module %q assigns llm_profile to deterministic validator %q", pipelineID, stage, chain.ModuleKey, validator.Module)
}
chain.Validators = append(chain.Validators, ResolvedValidator{
Binding: cloneModuleBinding(validator),
ExecutionClass: spec.ExecutionClass,
@@ -728,6 +762,7 @@ func resolveBinding(binding ModuleBinding, defaultModule string) ModuleBinding {
Retries: binding.Retries,
Options: cloneOptions(binding.Options),
References: normalizeReferenceMap(binding.References),
Validators: cloneValidatorOverride(binding.Validators),
}
}

146
internal/framework/pipeline/profile_test.go
View File

@@ -182,6 +182,143 @@ func TestResolvePipelineRejectsUnknownDefaultValidator(t *testing.T) {
}
}
func TestResolvePipelineValidatorOverrideReplacesDefaultChain(t *testing.T) {
catalog := newProfileCatalog(t)
registerProfileValidatorSpec(t, catalog, ValidatorSpec{Key: "second-validator", ExecutionClass: contracts.ExecutionClassLLMBacked})
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
Stage: StageExtract,
Module: "event-extractor",
Validators: []ModuleBinding{Binding("grounded")},
}); err != nil {
t.Fatalf("register validator chain: %v", err)
}
profile := PipelineProfile{
ID: "validated",
Input: Binding("text"),
Artifacts: map[string]ArtifactLaneProfile{
"events": {
Extract: ModuleBinding{
Module: "event-extractor",
Validators: ValidatorOverride{
Set: true,
Validators: []ModuleBinding{
{Module: "second-validator", LLMProfile: "careful"},
Binding("grounded"),
},
},
},
},
},
}
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
extractChain := findResolvedValidatorChain(resolved.ValidatorChains, StageExtract, "events", "event-extractor")
if extractChain == nil {
t.Fatal("extract validator chain not found")
}
if len(extractChain.Validators) != 2 {
t.Fatalf("extract validators = %#v, want explicit two-validator override", extractChain.Validators)
}
if extractChain.Validators[0].Binding.Module != "second-validator" || extractChain.Validators[0].Binding.LLMProfile != "careful" {
t.Fatalf("first validator = %#v, want explicit LLM-backed validator first", extractChain.Validators[0])
}
if extractChain.Validators[1].Binding.Module != "grounded" {
t.Fatalf("second validator = %#v, want grounded second", extractChain.Validators[1])
}
}
func TestResolvePipelineExplicitEmptyValidatorOverrideSuppressesDefaultChain(t *testing.T) {
catalog := newProfileCatalog(t)
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
Stage: StageExtract,
Module: "event-extractor",
Validators: []ModuleBinding{Binding("grounded")},
}); err != nil {
t.Fatalf("register validator chain: %v", err)
}
profile := PipelineProfile{
ID: "validated",
Input: Binding("text"),
Artifacts: map[string]ArtifactLaneProfile{
"events": {
Extract: ModuleBinding{
Module: "event-extractor",
Validators: ValidatorOverride{Set: true},
},
},
},
}
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
if err != nil {
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
}
extractChain := findResolvedValidatorChain(resolved.ValidatorChains, StageExtract, "events", "event-extractor")
if extractChain == nil {
t.Fatal("extract validator chain not found")
}
if len(extractChain.Validators) != 0 {
t.Fatalf("extract validators = %#v, want explicit empty override", extractChain.Validators)
}
}
func TestResolvePipelineRejectsUnknownOverrideValidator(t *testing.T) {
_, err := ResolvePipeline(PipelineProfile{
ID: "validated",
Input: Binding("text"),
Artifacts: map[string]ArtifactLaneProfile{
"events": {
Extract: ModuleBinding{
Module: "event-extractor",
Validators: ValidatorOverride{
Set: true,
Validators: []ModuleBinding{Binding("missing-validator")},
},
},
},
},
}, ResolveOptions{}, newProfileCatalog(t))
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want unknown validator error")
}
if !strings.Contains(err.Error(), "missing-validator") {
t.Fatalf("ResolvePipeline() error = %q, want missing validator context", err.Error())
}
}
func TestResolvePipelineRejectsLLMProfileForDeterministicValidator(t *testing.T) {
_, err := ResolvePipeline(PipelineProfile{
ID: "validated",
Input: Binding("text"),
Artifacts: map[string]ArtifactLaneProfile{
"events": {
Extract: ModuleBinding{
Module: "event-extractor",
Validators: ValidatorOverride{
Set: true,
Validators: []ModuleBinding{
{Module: "grounded", LLMProfile: "careful"},
},
},
},
},
},
}, ResolveOptions{}, newProfileCatalog(t))
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want deterministic validator profile error")
}
if !strings.Contains(err.Error(), "grounded") || !strings.Contains(err.Error(), "llm_profile") {
t.Fatalf("ResolvePipeline() error = %q, want validator profile context", err.Error())
}
}
func TestResolvePipelineSelectsOnlyRequestedLanes(t *testing.T) {
profile := multiLaneProfile()
resolved, err := ResolvePipeline(profile, ResolveOptions{Only: []string{" summaries ", "events", "summaries"}}, newProfileCatalog(t))
@@ -850,7 +987,7 @@ func TestResolvePipelineRejectsConfiguredValidators(t *testing.T) {
if err == nil {
t.Fatal("ResolvePipeline() error = nil, want error")
}
assertErrorContains(t, err, "baseline", "events", "configured validators", "not supported")
assertErrorContains(t, err, "baseline", "events", "validators", "extract.validators")
}
func TestResolvePipelineOrdersLanesDeterministically(t *testing.T) {
@@ -1139,6 +1276,13 @@ func registerProfileSpecs(t *testing.T, catalog ModuleCatalog, specs ...ModuleSp
}
}
func registerProfileValidatorSpec(t *testing.T, catalog ModuleCatalog, spec ValidatorSpec) {
t.Helper()
if err := catalog.Validators.RegisterWithSpec(spec, profileValidatorConstructor(spec.Key)); err != nil {
t.Fatalf("register validator spec %#v: %v", spec, err)
}
}
func profileInputConstructor(key string) InputAdapterConstructor {
return func() (contracts.InputAdapter, error) {
return profileInputAdapter{key: key}, nil

2
internal/framework/pipeline/runner.go
View File

@@ -631,7 +631,7 @@ func validateRunInput(input RunInput) error {
return fmt.Errorf("resolved pipeline lane %q normalize module must not be empty", lane.ID)
}
if len(lane.Validators) > 0 {
return fmt.Errorf("resolved pipeline lane %q configured validators are not supported by the current raw validation runner", lane.ID)
return fmt.Errorf("resolved pipeline lane %q validators are not supported at artifact lane level; use extract.validators, merge.validators, or normalize.validators", lane.ID)
}
}
return nil

2
internal/framework/pipeline/runner_test.go
View File

@@ -1081,7 +1081,7 @@ func TestRunRejectsConfiguredValidators(t *testing.T) {
_, err := New(newRunnerRegistries(t, defaultRunnerModules())).Run(context.Background(), RunInput{
Pipeline: resolvedPipelineWithValidators("configured", "second-validator"),
})
assertRunError(t, err, "configured validators")
assertRunError(t, err, "extract.validators")
}
func TestRunCollectsStageWarnings(t *testing.T) {

8
internal/framework/pipeline/validator_chain_registry.go
View File

@@ -98,5 +98,13 @@ func cloneModuleBinding(binding ModuleBinding) ModuleBinding {
}
binding.References = references
}
binding.Validators = cloneValidatorOverride(binding.Validators)
return binding
}
func cloneValidatorOverride(override ValidatorOverride) ValidatorOverride {
return ValidatorOverride{
Set: override.Set,
Validators: cloneModuleBindings(override.Validators),
}
}