Add validator chain config overrides
This commit is contained in:
@@ -45,8 +45,9 @@ Flags:
|
|||||||
Defaults to `./notarius-output`.
|
Defaults to `./notarius-output`.
|
||||||
- `--diagnostics-dir path`: diagnostics work directory override for this
|
- `--diagnostics-dir path`: diagnostics work directory override for this
|
||||||
invocation.
|
invocation.
|
||||||
- `--llm-profile id`: override every effective LLM-capable module binding to
|
- `--llm-profile id`: override every effective LLM-capable pipeline module
|
||||||
use one Scriptorium profile ID.
|
binding to use one Scriptorium profile ID. Validator-specific profiles are
|
||||||
|
not overridden.
|
||||||
- `--session-id id`: pass a stable prompt session identifier through LLM-backed
|
- `--session-id id`: pass a stable prompt session identifier through LLM-backed
|
||||||
module calls.
|
module calls.
|
||||||
- `--reference selector=path`: bind a reference path to a chunk, extractor,
|
- `--reference selector=path`: bind a reference path to a chunk, extractor,
|
||||||
|
|||||||
@@ -131,8 +131,9 @@ Artifact lane fields:
|
|||||||
- `extract`: required module binding.
|
- `extract`: required module binding.
|
||||||
- `merge`: optional module binding. Default module is `appendorder`.
|
- `merge`: optional module binding. Default module is `appendorder`.
|
||||||
- `normalize`: optional module binding. Default module is `noop`.
|
- `normalize`: optional module binding. Default module is `noop`.
|
||||||
- `validators`: reserved for future configurable validator chains. Non-empty
|
- `validators`: deprecated lane-level validator list. Non-empty lists are
|
||||||
lists are rejected by current configuration validation.
|
rejected; use `extract.validators`, `merge.validators`, or
|
||||||
|
`normalize.validators`.
|
||||||
- `references`: optional compatibility alias for extractor reference bindings.
|
- `references`: optional compatibility alias for extractor reference bindings.
|
||||||
Lane bindings override pipeline-level bindings for the same slot.
|
Lane bindings override pipeline-level bindings for the same slot.
|
||||||
|
|
||||||
@@ -234,12 +235,28 @@ Binding fields:
|
|||||||
- `options`: optional module-specific settings.
|
- `options`: optional module-specific settings.
|
||||||
- `references`: optional reference bindings. Supported only for `chunk`,
|
- `references`: optional reference bindings. Supported only for `chunk`,
|
||||||
`extract`, `merge`, and `normalize` bindings. `input` and `output` bindings
|
`extract`, `merge`, and `normalize` bindings. `input` and `output` bindings
|
||||||
reject this field during validation. Validator bindings are reserved for a
|
reject this field during validation.
|
||||||
future validator-chain feature and are rejected when configured.
|
- `validators`: optional stage-local validator chain override. Supported only
|
||||||
|
for `chunk`, `extract`, `merge`, and `normalize` bindings. Omit the field to
|
||||||
|
use the production default chain; set `validators: []` to force an empty
|
||||||
|
chain; set a non-empty list to use exactly those validators in configured
|
||||||
|
order.
|
||||||
|
|
||||||
|
Validator bindings use the same shorthand or object module-binding form, but
|
||||||
|
only these fields are supported:
|
||||||
|
|
||||||
|
- `module`: validator key.
|
||||||
|
- `llm_profile`: optional Scriptorium profile ID for LLM-backed validators.
|
||||||
|
- `options`: optional validator-specific settings.
|
||||||
|
|
||||||
|
Validator bindings reject `references`, `retries`, and nested `validators`.
|
||||||
|
During resolution, deterministic validators reject explicit `llm_profile`
|
||||||
|
values.
|
||||||
|
|
||||||
The `--llm-profile` run flag overrides every effective LLM-capable module
|
The `--llm-profile` run flag overrides every effective LLM-capable module
|
||||||
binding to use one Scriptorium profile ID: chunk, every selected lane extract,
|
binding to use one Scriptorium profile ID: chunk, every selected lane extract,
|
||||||
merge, and normalize binding.
|
merge, and normalize binding. It does not override validator-specific
|
||||||
|
`llm_profile` values.
|
||||||
|
|
||||||
## Implemented Production Modules
|
## Implemented Production Modules
|
||||||
|
|
||||||
|
|||||||
@@ -187,10 +187,10 @@ Runner-side raw validation chains receive the raw module output plus
|
|||||||
stage, lane, module, source, and chunk provenance. Empty raw validation chains
|
stage, lane, module, source, and chunk provenance. Empty raw validation chains
|
||||||
approve output by default.
|
approve output by default.
|
||||||
|
|
||||||
Pipeline-configured validator lists are not part of the current runner
|
Resolved validator chains come from central default mappings unless a
|
||||||
contract. Non-empty configured validator lists are rejected during configuration
|
stage-local config override is set on `chunk`, lane `extract`, lane `merge`, or
|
||||||
validation or resolved-run validation so they cannot appear in manifests without
|
lane `normalize`. Explicit empty overrides are valid and are recorded as empty
|
||||||
executing.
|
chains in manifests.
|
||||||
|
|
||||||
Validator rejection is a non-fatal run outcome: the rejected output is recorded
|
Validator rejection is a non-fatal run outcome: the rejected output is recorded
|
||||||
in `RunOutput.Rejected` and does not pass to the next stage. Validator execution
|
in `RunOutput.Rejected` and does not pass to the next stage. Validator execution
|
||||||
|
|||||||
@@ -495,6 +495,13 @@ func effectiveLLMProfileIDs(resolved pipeline.ResolvedPipeline) []string {
|
|||||||
add(lane.Merge)
|
add(lane.Merge)
|
||||||
add(lane.Normalize)
|
add(lane.Normalize)
|
||||||
}
|
}
|
||||||
|
for _, chain := range resolved.ValidatorChains {
|
||||||
|
for _, validator := range chain.Validators {
|
||||||
|
if validator.ExecutionClass == contracts.ExecutionClassLLMBacked {
|
||||||
|
add(validator.Binding)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
ids := make([]string, 0, len(seen))
|
ids := make([]string, 0, len(seen))
|
||||||
for id := range seen {
|
for id := range seen {
|
||||||
ids = append(ids, id)
|
ids = append(ids, id)
|
||||||
|
|||||||
@@ -861,13 +861,29 @@ func TestEffectiveLLMProfileIDsUsesLLMCapableStagesOnly(t *testing.T) {
|
|||||||
Extract: pipeline.ModuleBinding{LLMProfile: "extract-profile"},
|
Extract: pipeline.ModuleBinding{LLMProfile: "extract-profile"},
|
||||||
Merge: pipeline.ModuleBinding{LLMProfile: "merge-profile"},
|
Merge: pipeline.ModuleBinding{LLMProfile: "merge-profile"},
|
||||||
Normalize: pipeline.ModuleBinding{LLMProfile: "normalize-profile"},
|
Normalize: pipeline.ModuleBinding{LLMProfile: "normalize-profile"},
|
||||||
Validators: []pipeline.ModuleBinding{{LLMProfile: "validator-profile"}},
|
},
|
||||||
|
},
|
||||||
|
ValidatorChains: []pipeline.ResolvedValidatorChain{
|
||||||
|
{
|
||||||
|
Stage: pipeline.StageExtract,
|
||||||
|
LaneID: "events",
|
||||||
|
ModuleKey: "extract",
|
||||||
|
Validators: []pipeline.ResolvedValidator{
|
||||||
|
{
|
||||||
|
Binding: pipeline.ModuleBinding{Module: "deterministic-validator", LLMProfile: "ignored-validator-profile"},
|
||||||
|
ExecutionClass: contracts.ExecutionClassDeterministic,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Binding: pipeline.ModuleBinding{Module: "llm-validator", LLMProfile: "validator-profile"},
|
||||||
|
ExecutionClass: contracts.ExecutionClassLLMBacked,
|
||||||
|
},
|
||||||
|
},
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
got := effectiveLLMProfileIDs(resolved)
|
got := effectiveLLMProfileIDs(resolved)
|
||||||
want := []string{"chunk-profile", "extract-profile", "merge-profile", "normalize-profile"}
|
want := []string{"chunk-profile", "extract-profile", "merge-profile", "normalize-profile", "validator-profile"}
|
||||||
if !reflect.DeepEqual(got, want) {
|
if !reflect.DeepEqual(got, want) {
|
||||||
t.Fatalf("effectiveLLMProfileIDs() = %#v, want %#v", got, want)
|
t.Fatalf("effectiveLLMProfileIDs() = %#v, want %#v", got, want)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -97,6 +97,18 @@ func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
|
|||||||
out.Options = cloneOptions(in.Options)
|
out.Options = cloneOptions(in.Options)
|
||||||
}
|
}
|
||||||
out.References = cloneStringMap(in.References)
|
out.References = cloneStringMap(in.References)
|
||||||
|
out.Validators = cloneValidatorOverride(in.Validators)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func cloneValidatorOverride(in pipeline.ValidatorOverride) pipeline.ValidatorOverride {
|
||||||
|
out := pipeline.ValidatorOverride{Set: in.Set}
|
||||||
|
if len(in.Validators) > 0 {
|
||||||
|
out.Validators = make([]pipeline.ModuleBinding, len(in.Validators))
|
||||||
|
for i, binding := range in.Validators {
|
||||||
|
out.Validators[i] = cloneModuleBinding(binding)
|
||||||
|
}
|
||||||
|
}
|
||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -161,6 +161,12 @@ func TestResolveLLMProfileOverrideAppliesBeforeDigest(t *testing.T) {
|
|||||||
profile.Output.LLMProfile = "output-profile"
|
profile.Output.LLMProfile = "output-profile"
|
||||||
lane := profile.Artifacts["events"]
|
lane := profile.Artifacts["events"]
|
||||||
lane.Merge.LLMProfile = "merge-profile"
|
lane.Merge.LLMProfile = "merge-profile"
|
||||||
|
lane.Extract.Validators = pipeline.ValidatorOverride{
|
||||||
|
Set: true,
|
||||||
|
Validators: []pipeline.ModuleBinding{
|
||||||
|
{Module: "fake/llm-validator", LLMProfile: "validator-profile"},
|
||||||
|
},
|
||||||
|
}
|
||||||
profile.Artifacts["events"] = lane
|
profile.Artifacts["events"] = lane
|
||||||
cfg.Pipelines["example"] = profile
|
cfg.Pipelines["example"] = profile
|
||||||
|
|
||||||
@@ -195,9 +201,22 @@ func TestResolveLLMProfileOverrideAppliesBeforeDigest(t *testing.T) {
|
|||||||
if eventLane.Merge.LLMProfile != "runtime" {
|
if eventLane.Merge.LLMProfile != "runtime" {
|
||||||
t.Fatalf("merge profile = %q, want runtime", eventLane.Merge.LLMProfile)
|
t.Fatalf("merge profile = %q, want runtime", eventLane.Merge.LLMProfile)
|
||||||
}
|
}
|
||||||
if len(eventLane.Validators) != 0 {
|
validatorChain := findEffectiveValidatorChain(effective.ResolvedPipeline.ValidatorChains, pipeline.StageExtract, "events", "fake/extract")
|
||||||
t.Fatalf("validator profiles = %#v, want none", eventLane.Validators)
|
if validatorChain == nil || len(validatorChain.Validators) != 1 {
|
||||||
|
t.Fatalf("validator chain = %#v, want one extract validator", effective.ResolvedPipeline.ValidatorChains)
|
||||||
}
|
}
|
||||||
|
if validatorChain.Validators[0].Binding.LLMProfile != "validator-profile" {
|
||||||
|
t.Fatalf("validator profile = %q, want original validator-profile", validatorChain.Validators[0].Binding.LLMProfile)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func findEffectiveValidatorChain(chains []pipeline.ResolvedValidatorChain, stage pipeline.ModuleStage, laneID string, module string) *pipeline.ResolvedValidatorChain {
|
||||||
|
for i := range chains {
|
||||||
|
if chains[i].Stage == stage && chains[i].LaneID == laneID && chains[i].ModuleKey == module {
|
||||||
|
return &chains[i]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func llmCapableBindings(resolved pipeline.ResolvedPipeline) []pipeline.ModuleBinding {
|
func llmCapableBindings(resolved pipeline.ResolvedPipeline) []pipeline.ModuleBinding {
|
||||||
|
|||||||
@@ -56,6 +56,7 @@ type fileModuleBinding struct {
|
|||||||
Retries int
|
Retries int
|
||||||
Options map[string]any
|
Options map[string]any
|
||||||
References map[string]string
|
References map[string]string
|
||||||
|
Validators pipeline.ValidatorOverride
|
||||||
}
|
}
|
||||||
|
|
||||||
func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
|
func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
|
||||||
@@ -102,6 +103,16 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
b.References = references
|
b.References = references
|
||||||
|
case "validators":
|
||||||
|
b.Validators.Set = true
|
||||||
|
var validators []fileModuleBinding
|
||||||
|
if err := valueNode.Decode(&validators); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
b.Validators.Validators = make([]pipeline.ModuleBinding, len(validators))
|
||||||
|
for i, validator := range validators {
|
||||||
|
b.Validators.Validators[i] = validator.toPipelineBinding()
|
||||||
|
}
|
||||||
default:
|
default:
|
||||||
return fmt.Errorf("field %s not found in module binding", keyNode.Value)
|
return fmt.Errorf("field %s not found in module binding", keyNode.Value)
|
||||||
}
|
}
|
||||||
@@ -119,6 +130,7 @@ func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
|
|||||||
Retries: b.Retries,
|
Retries: b.Retries,
|
||||||
Options: cloneOptions(b.Options),
|
Options: cloneOptions(b.Options),
|
||||||
References: normalizedStringMap(b.References),
|
References: normalizedStringMap(b.References),
|
||||||
|
Validators: b.Validators,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -300,6 +300,61 @@ pipelines:
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestParseFileConfigStageLocalValidatorOverrides(t *testing.T) {
|
||||||
|
cfg := parseAndApplyConfig(t, `
|
||||||
|
version: 2
|
||||||
|
pipelines:
|
||||||
|
example:
|
||||||
|
input: fake/input
|
||||||
|
chunk:
|
||||||
|
module: generic
|
||||||
|
validators: []
|
||||||
|
artifacts:
|
||||||
|
events:
|
||||||
|
extract:
|
||||||
|
module: fake/extract
|
||||||
|
validators:
|
||||||
|
- fake/validator
|
||||||
|
- module: fake/llm-validator
|
||||||
|
llm_profile: careful
|
||||||
|
options:
|
||||||
|
threshold: 0.7
|
||||||
|
merge:
|
||||||
|
module: appendorder
|
||||||
|
validators: []
|
||||||
|
normalize:
|
||||||
|
module: noop
|
||||||
|
`)
|
||||||
|
|
||||||
|
profile := cfg.Pipelines["example"]
|
||||||
|
if !profile.Chunk.Validators.Set || len(profile.Chunk.Validators.Validators) != 0 {
|
||||||
|
t.Fatalf("chunk validator override = %#v, want explicit empty", profile.Chunk.Validators)
|
||||||
|
}
|
||||||
|
lane := profile.Artifacts["events"]
|
||||||
|
if !lane.Extract.Validators.Set {
|
||||||
|
t.Fatalf("extract validator override Set = false, want true")
|
||||||
|
}
|
||||||
|
validators := lane.Extract.Validators.Validators
|
||||||
|
if len(validators) != 2 {
|
||||||
|
t.Fatalf("extract validators = %#v, want two validators", validators)
|
||||||
|
}
|
||||||
|
if validators[0].Module != "fake/validator" {
|
||||||
|
t.Fatalf("first validator = %#v, want fake/validator", validators[0])
|
||||||
|
}
|
||||||
|
if validators[1].Module != "fake/llm-validator" || validators[1].LLMProfile != "careful" {
|
||||||
|
t.Fatalf("second validator = %#v, want LLM validator with profile", validators[1])
|
||||||
|
}
|
||||||
|
if validators[1].Options["threshold"] != 0.7 {
|
||||||
|
t.Fatalf("second validator options = %#v, want threshold", validators[1].Options)
|
||||||
|
}
|
||||||
|
if !lane.Merge.Validators.Set || len(lane.Merge.Validators.Validators) != 0 {
|
||||||
|
t.Fatalf("merge validator override = %#v, want explicit empty", lane.Merge.Validators)
|
||||||
|
}
|
||||||
|
if lane.Normalize.Validators.Set {
|
||||||
|
t.Fatalf("normalize validator override Set = true, want omitted")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestApplyFileConfigRejectsDuplicateTrimmedPipelineIDs(t *testing.T) {
|
func TestApplyFileConfigRejectsDuplicateTrimmedPipelineIDs(t *testing.T) {
|
||||||
fileCfg, err := ParseFileConfigYAML([]byte(`
|
fileCfg, err := ParseFileConfigYAML([]byte(`
|
||||||
version: 2
|
version: 2
|
||||||
|
|||||||
@@ -27,6 +27,12 @@ func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeli
|
|||||||
out.Chunk = cloneModuleBinding(in.Chunk)
|
out.Chunk = cloneModuleBinding(in.Chunk)
|
||||||
out.ChunkReferences = pipeline.CloneReferenceTarget(in.ChunkReferences)
|
out.ChunkReferences = pipeline.CloneReferenceTarget(in.ChunkReferences)
|
||||||
out.Output = cloneModuleBinding(in.Output)
|
out.Output = cloneModuleBinding(in.Output)
|
||||||
|
if len(in.ValidatorChains) > 0 {
|
||||||
|
out.ValidatorChains = make([]pipeline.ResolvedValidatorChain, len(in.ValidatorChains))
|
||||||
|
for i, chain := range in.ValidatorChains {
|
||||||
|
out.ValidatorChains[i] = cloneResolvedValidatorChain(chain)
|
||||||
|
}
|
||||||
|
}
|
||||||
if len(in.ArtifactLanes) > 0 {
|
if len(in.ArtifactLanes) > 0 {
|
||||||
out.ArtifactLanes = make([]pipeline.ResolvedArtifactLane, len(in.ArtifactLanes))
|
out.ArtifactLanes = make([]pipeline.ResolvedArtifactLane, len(in.ArtifactLanes))
|
||||||
for i, lane := range in.ArtifactLanes {
|
for i, lane := range in.ArtifactLanes {
|
||||||
@@ -36,6 +42,20 @@ func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeli
|
|||||||
return out
|
return out
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func cloneResolvedValidatorChain(in pipeline.ResolvedValidatorChain) pipeline.ResolvedValidatorChain {
|
||||||
|
out := in
|
||||||
|
if len(in.Validators) > 0 {
|
||||||
|
out.Validators = make([]pipeline.ResolvedValidator, len(in.Validators))
|
||||||
|
for i, validator := range in.Validators {
|
||||||
|
out.Validators[i] = pipeline.ResolvedValidator{
|
||||||
|
Binding: cloneModuleBinding(validator.Binding),
|
||||||
|
ExecutionClass: validator.ExecutionClass,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
func cloneResolvedArtifactLane(in pipeline.ResolvedArtifactLane) pipeline.ResolvedArtifactLane {
|
func cloneResolvedArtifactLane(in pipeline.ResolvedArtifactLane) pipeline.ResolvedArtifactLane {
|
||||||
out := in
|
out := in
|
||||||
out.Extract = cloneModuleBinding(in.Extract)
|
out.Extract = cloneModuleBinding(in.Extract)
|
||||||
|
|||||||
@@ -85,7 +85,7 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile) erro
|
|||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if len(lane.Validators) > 0 {
|
if len(lane.Validators) > 0 {
|
||||||
return fmt.Errorf("pipeline %q lane %q validators are not supported by the current raw validation runner", id, laneID)
|
return fmt.Errorf("pipeline %q lane %q validators are not supported at artifact lane level; use extract.validators, merge.validators, or normalize.validators", id, laneID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -108,6 +108,9 @@ func validateBinding(
|
|||||||
}
|
}
|
||||||
return fmt.Errorf("pipeline %q %s retries must be greater than or equal to zero", pipelineID, slot)
|
return fmt.Errorf("pipeline %q %s retries must be greater than or equal to zero", pipelineID, slot)
|
||||||
}
|
}
|
||||||
|
if err := validateValidatorOverride(pipelineID, laneID, slot, binding.Validators); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
if len(binding.References) == 0 {
|
if len(binding.References) == 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@@ -120,6 +123,36 @@ func validateBinding(
|
|||||||
return validateReferenceMapForContext(pipelineID, laneID, slot, binding.References)
|
return validateReferenceMapForContext(pipelineID, laneID, slot, binding.References)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func validateValidatorOverride(pipelineID string, laneID string, slot string, override pipeline.ValidatorOverride) error {
|
||||||
|
if !override.Set {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
switch slot {
|
||||||
|
case "chunk", "extract", "merge", "normalize":
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("%s validators are not supported", referenceContext(pipelineID, laneID, slot))
|
||||||
|
}
|
||||||
|
for i, validator := range override.Validators {
|
||||||
|
context := fmt.Sprintf("%s validators[%d]", referenceContext(pipelineID, laneID, slot), i)
|
||||||
|
if strings.TrimSpace(validator.Module) == "" {
|
||||||
|
return fmt.Errorf("%s module must not be empty", context)
|
||||||
|
}
|
||||||
|
if len(validator.References) > 0 {
|
||||||
|
return fmt.Errorf("%s references are not supported", context)
|
||||||
|
}
|
||||||
|
if validator.Validators.Set {
|
||||||
|
return fmt.Errorf("%s nested validators are not supported", context)
|
||||||
|
}
|
||||||
|
if validator.Retries != 0 {
|
||||||
|
return fmt.Errorf("%s retries are not supported", context)
|
||||||
|
}
|
||||||
|
if validator.LLMProfile != "" && strings.TrimSpace(validator.LLMProfile) == "" {
|
||||||
|
return fmt.Errorf("%s llm_profile must not be empty when set", context)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
func validateReferenceMap(pipelineID string, laneID string, references map[string]string) error {
|
func validateReferenceMap(pipelineID string, laneID string, references map[string]string) error {
|
||||||
return validateReferenceMapForContext(pipelineID, laneID, "", references)
|
return validateReferenceMapForContext(pipelineID, laneID, "", references)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -341,13 +341,82 @@ func TestValidateRejectsConfiguredValidators(t *testing.T) {
|
|||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("Validate() error = nil, want configured validators error")
|
t.Fatal("Validate() error = nil, want configured validators error")
|
||||||
}
|
}
|
||||||
for _, want := range []string{"example", "events", "validators", "not supported"} {
|
for _, want := range []string{"example", "events", "validators", "extract.validators", "merge.validators", "normalize.validators"} {
|
||||||
if !strings.Contains(err.Error(), want) {
|
if !strings.Contains(err.Error(), want) {
|
||||||
t.Fatalf("Validate() error = %q, want substring %q", err.Error(), want)
|
t.Fatalf("Validate() error = %q, want substring %q", err.Error(), want)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestValidateAcceptsStageLocalValidatorOverrides(t *testing.T) {
|
||||||
|
cfg := validConfig()
|
||||||
|
profile := cfg.Pipelines["example"]
|
||||||
|
profile.Chunk.Validators = pipeline.ValidatorOverride{Set: true}
|
||||||
|
lane := profile.Artifacts["events"]
|
||||||
|
lane.Extract.Validators = pipeline.ValidatorOverride{
|
||||||
|
Set: true,
|
||||||
|
Validators: []pipeline.ModuleBinding{
|
||||||
|
pipeline.Binding("fake/validator"),
|
||||||
|
{Module: "fake/llm-validator", LLMProfile: "careful", Options: map[string]any{"threshold": 0.7}},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
lane.Merge.Validators = pipeline.ValidatorOverride{Set: true}
|
||||||
|
profile.Artifacts["events"] = lane
|
||||||
|
cfg.Pipelines["example"] = profile
|
||||||
|
|
||||||
|
if err := cfg.Validate(); err != nil {
|
||||||
|
t.Fatalf("Validate() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestValidateRejectsInvalidValidatorBindings(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
binding pipeline.ModuleBinding
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "empty module",
|
||||||
|
binding: pipeline.ModuleBinding{},
|
||||||
|
want: "module must not be empty",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "references",
|
||||||
|
binding: pipeline.ModuleBinding{Module: "fake/validator", References: map[string]string{"roster": "./roster.txt"}},
|
||||||
|
want: "references are not supported",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "nested validators",
|
||||||
|
binding: pipeline.ModuleBinding{Module: "fake/validator", Validators: pipeline.ValidatorOverride{Set: true}},
|
||||||
|
want: "nested validators are not supported",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "retries",
|
||||||
|
binding: pipeline.ModuleBinding{Module: "fake/validator", Retries: 1},
|
||||||
|
want: "retries are not supported",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
t.Run(test.name, func(t *testing.T) {
|
||||||
|
cfg := validConfig()
|
||||||
|
profile := cfg.Pipelines["example"]
|
||||||
|
lane := profile.Artifacts["events"]
|
||||||
|
lane.Extract.Validators = pipeline.ValidatorOverride{
|
||||||
|
Set: true,
|
||||||
|
Validators: []pipeline.ModuleBinding{test.binding},
|
||||||
|
}
|
||||||
|
profile.Artifacts["events"] = lane
|
||||||
|
cfg.Pipelines["example"] = profile
|
||||||
|
|
||||||
|
err := cfg.Validate()
|
||||||
|
if err == nil || !strings.Contains(err.Error(), test.want) {
|
||||||
|
t.Fatalf("Validate() error = %v, want %q", err, test.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func validConfig() Config {
|
func validConfig() Config {
|
||||||
cfg := Default()
|
cfg := Default()
|
||||||
cfg.Pipelines["example"] = pipeline.PipelineProfile{
|
cfg.Pipelines["example"] = pipeline.PipelineProfile{
|
||||||
@@ -402,6 +471,12 @@ func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.Module
|
|||||||
Requires: []string{"normalized"},
|
Requires: []string{"normalized"},
|
||||||
Provides: []string{"validated"},
|
Provides: []string{"validated"},
|
||||||
},
|
},
|
||||||
|
"fake/llm-validator": {
|
||||||
|
Key: "fake/llm-validator",
|
||||||
|
Stage: pipeline.StageValidate,
|
||||||
|
Requires: []string{"normalized"},
|
||||||
|
Provides: []string{"validated"},
|
||||||
|
},
|
||||||
"json": {
|
"json": {
|
||||||
Key: "json",
|
Key: "json",
|
||||||
Stage: pipeline.StageOutput,
|
Stage: pipeline.StageOutput,
|
||||||
@@ -426,6 +501,7 @@ func fakeCatalog(t *testing.T, overrides ...pipeline.ModuleSpec) pipeline.Module
|
|||||||
mustRegisterMerger(t, mergers, specs["appendorder"])
|
mustRegisterMerger(t, mergers, specs["appendorder"])
|
||||||
mustRegisterNormalizer(t, normalizers, specs["noop"])
|
mustRegisterNormalizer(t, normalizers, specs["noop"])
|
||||||
mustRegisterValidator(t, validators, specs["fake/validator"])
|
mustRegisterValidator(t, validators, specs["fake/validator"])
|
||||||
|
mustRegisterValidator(t, validators, specs["fake/llm-validator"])
|
||||||
mustRegisterOutput(t, outputs, specs["json"])
|
mustRegisterOutput(t, outputs, specs["json"])
|
||||||
|
|
||||||
return pipeline.ModuleCatalog{
|
return pipeline.ModuleCatalog{
|
||||||
@@ -477,7 +553,11 @@ func mustRegisterNormalizer(t *testing.T, registry *pipeline.NormalizerRegistry,
|
|||||||
|
|
||||||
func mustRegisterValidator(t *testing.T, registry *pipeline.ValidatorRegistry, spec pipeline.ModuleSpec) {
|
func mustRegisterValidator(t *testing.T, registry *pipeline.ValidatorRegistry, spec pipeline.ModuleSpec) {
|
||||||
t.Helper()
|
t.Helper()
|
||||||
validatorSpec := pipeline.ValidatorSpec{Key: spec.Key, ExecutionClass: contracts.ExecutionClassDeterministic}
|
executionClass := contracts.ExecutionClassDeterministic
|
||||||
|
if spec.Key == "fake/llm-validator" {
|
||||||
|
executionClass = contracts.ExecutionClassLLMBacked
|
||||||
|
}
|
||||||
|
validatorSpec := pipeline.ValidatorSpec{Key: spec.Key, ExecutionClass: executionClass}
|
||||||
if err := registry.RegisterWithSpec(validatorSpec, func() (contracts.Validator, error) { return nil, nil }); err != nil {
|
if err := registry.RegisterWithSpec(validatorSpec, func() (contracts.Validator, error) { return nil, nil }); err != nil {
|
||||||
t.Fatalf("register validator: %v", err)
|
t.Fatalf("register validator: %v", err)
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -25,6 +25,35 @@ type ModuleBinding struct {
|
|||||||
Retries int `json:"retries,omitempty"`
|
Retries int `json:"retries,omitempty"`
|
||||||
Options map[string]any `json:"options,omitempty"`
|
Options map[string]any `json:"options,omitempty"`
|
||||||
References map[string]string `json:"references,omitempty"`
|
References map[string]string `json:"references,omitempty"`
|
||||||
|
Validators ValidatorOverride `json:"validators,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type ValidatorOverride struct {
|
||||||
|
Set bool `json:"set,omitempty"`
|
||||||
|
Validators []ModuleBinding `json:"validators,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (binding ModuleBinding) MarshalJSON() ([]byte, error) {
|
||||||
|
type moduleBindingJSON struct {
|
||||||
|
Module string `json:"module"`
|
||||||
|
LLMProfile string `json:"llm_profile,omitempty"`
|
||||||
|
Retries int `json:"retries,omitempty"`
|
||||||
|
Options map[string]any `json:"options,omitempty"`
|
||||||
|
References map[string]string `json:"references,omitempty"`
|
||||||
|
Validators *[]ModuleBinding `json:"validators,omitempty"`
|
||||||
|
}
|
||||||
|
out := moduleBindingJSON{
|
||||||
|
Module: binding.Module,
|
||||||
|
LLMProfile: binding.LLMProfile,
|
||||||
|
Retries: binding.Retries,
|
||||||
|
Options: binding.Options,
|
||||||
|
References: binding.References,
|
||||||
|
}
|
||||||
|
if binding.Validators.Set {
|
||||||
|
validators := cloneModuleBindings(binding.Validators.Validators)
|
||||||
|
out.Validators = &validators
|
||||||
|
}
|
||||||
|
return json.Marshal(out)
|
||||||
}
|
}
|
||||||
|
|
||||||
type ArtifactLaneProfile struct {
|
type ArtifactLaneProfile struct {
|
||||||
@@ -186,7 +215,7 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
|
|||||||
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences),
|
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences),
|
||||||
Output: resolveBinding(profile.Output, DefaultOutputModule),
|
Output: resolveBinding(profile.Output, DefaultOutputModule),
|
||||||
}
|
}
|
||||||
chunkValidatorChain, err := resolveValidatorChain(pipelineID, "", StageChunk, chunk.Module, catalog)
|
chunkValidatorChain, err := resolveValidatorChain(pipelineID, "", StageChunk, chunk.Module, chunk.Validators, catalog)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ResolvedPipeline{}, err
|
return ResolvedPipeline{}, err
|
||||||
}
|
}
|
||||||
@@ -316,15 +345,15 @@ func resolveArtifactLane(
|
|||||||
return ResolvedArtifactLane{}, nil, nil, configuredValidatorsError(pipelineID, laneID)
|
return ResolvedArtifactLane{}, nil, nil, configuredValidatorsError(pipelineID, laneID)
|
||||||
}
|
}
|
||||||
|
|
||||||
extractValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageExtract, lane.Extract.Module, catalog)
|
extractValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageExtract, lane.Extract.Module, lane.Extract.Validators, catalog)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ResolvedArtifactLane{}, nil, nil, err
|
return ResolvedArtifactLane{}, nil, nil, err
|
||||||
}
|
}
|
||||||
mergeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageMerge, lane.Merge.Module, catalog)
|
mergeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageMerge, lane.Merge.Module, lane.Merge.Validators, catalog)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ResolvedArtifactLane{}, nil, nil, err
|
return ResolvedArtifactLane{}, nil, nil, err
|
||||||
}
|
}
|
||||||
normalizeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageNormalize, lane.Normalize.Module, catalog)
|
normalizeValidatorChain, err := resolveValidatorChain(pipelineID, laneID, StageNormalize, lane.Normalize.Module, lane.Normalize.Validators, catalog)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return ResolvedArtifactLane{}, nil, nil, err
|
return ResolvedArtifactLane{}, nil, nil, err
|
||||||
}
|
}
|
||||||
@@ -334,10 +363,10 @@ func resolveArtifactLane(
|
|||||||
}
|
}
|
||||||
|
|
||||||
func configuredValidatorsError(pipelineID string, laneID string) error {
|
func configuredValidatorsError(pipelineID string, laneID string) error {
|
||||||
return fmt.Errorf("pipeline %q lane %q configured validators are not supported by the current raw validation runner", pipelineID, laneID)
|
return fmt.Errorf("pipeline %q lane %q validators are not supported at artifact lane level; use extract.validators, merge.validators, or normalize.validators", pipelineID, laneID)
|
||||||
}
|
}
|
||||||
|
|
||||||
func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage, module string, catalog ModuleCatalog) (ResolvedValidatorChain, error) {
|
func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage, module string, override ValidatorOverride, catalog ModuleCatalog) (ResolvedValidatorChain, error) {
|
||||||
chain := ResolvedValidatorChain{
|
chain := ResolvedValidatorChain{
|
||||||
Stage: stage,
|
Stage: stage,
|
||||||
LaneID: strings.TrimSpace(laneID),
|
LaneID: strings.TrimSpace(laneID),
|
||||||
@@ -352,10 +381,12 @@ func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage,
|
|||||||
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q validator chain stage %q is not supported", pipelineID, stage)
|
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q validator chain stage %q is not supported", pipelineID, stage)
|
||||||
}
|
}
|
||||||
|
|
||||||
if catalog.ValidatorChains == nil {
|
var bindings []ModuleBinding
|
||||||
return chain, nil
|
if override.Set {
|
||||||
|
bindings = cloneModuleBindings(override.Validators)
|
||||||
|
} else if catalog.ValidatorChains != nil {
|
||||||
|
bindings = catalog.ValidatorChains.Validators(stage, chain.ModuleKey)
|
||||||
}
|
}
|
||||||
bindings := catalog.ValidatorChains.Validators(stage, chain.ModuleKey)
|
|
||||||
if len(bindings) == 0 {
|
if len(bindings) == 0 {
|
||||||
return chain, nil
|
return chain, nil
|
||||||
}
|
}
|
||||||
@@ -368,6 +399,9 @@ func resolveValidatorChain(pipelineID string, laneID string, stage ModuleStage,
|
|||||||
if !ok {
|
if !ok {
|
||||||
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q %s validator chain for module %q references unknown validator %q", pipelineID, stage, chain.ModuleKey, validator.Module)
|
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q %s validator chain for module %q references unknown validator %q", pipelineID, stage, chain.ModuleKey, validator.Module)
|
||||||
}
|
}
|
||||||
|
if strings.TrimSpace(validator.LLMProfile) != "" && spec.ExecutionClass != contracts.ExecutionClassLLMBacked {
|
||||||
|
return ResolvedValidatorChain{}, fmt.Errorf("pipeline %q %s validator chain for module %q assigns llm_profile to deterministic validator %q", pipelineID, stage, chain.ModuleKey, validator.Module)
|
||||||
|
}
|
||||||
chain.Validators = append(chain.Validators, ResolvedValidator{
|
chain.Validators = append(chain.Validators, ResolvedValidator{
|
||||||
Binding: cloneModuleBinding(validator),
|
Binding: cloneModuleBinding(validator),
|
||||||
ExecutionClass: spec.ExecutionClass,
|
ExecutionClass: spec.ExecutionClass,
|
||||||
@@ -728,6 +762,7 @@ func resolveBinding(binding ModuleBinding, defaultModule string) ModuleBinding {
|
|||||||
Retries: binding.Retries,
|
Retries: binding.Retries,
|
||||||
Options: cloneOptions(binding.Options),
|
Options: cloneOptions(binding.Options),
|
||||||
References: normalizeReferenceMap(binding.References),
|
References: normalizeReferenceMap(binding.References),
|
||||||
|
Validators: cloneValidatorOverride(binding.Validators),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -182,6 +182,143 @@ func TestResolvePipelineRejectsUnknownDefaultValidator(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineValidatorOverrideReplacesDefaultChain(t *testing.T) {
|
||||||
|
catalog := newProfileCatalog(t)
|
||||||
|
registerProfileValidatorSpec(t, catalog, ValidatorSpec{Key: "second-validator", ExecutionClass: contracts.ExecutionClassLLMBacked})
|
||||||
|
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
|
||||||
|
Stage: StageExtract,
|
||||||
|
Module: "event-extractor",
|
||||||
|
Validators: []ModuleBinding{Binding("grounded")},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("register validator chain: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
profile := PipelineProfile{
|
||||||
|
ID: "validated",
|
||||||
|
Input: Binding("text"),
|
||||||
|
Artifacts: map[string]ArtifactLaneProfile{
|
||||||
|
"events": {
|
||||||
|
Extract: ModuleBinding{
|
||||||
|
Module: "event-extractor",
|
||||||
|
Validators: ValidatorOverride{
|
||||||
|
Set: true,
|
||||||
|
Validators: []ModuleBinding{
|
||||||
|
{Module: "second-validator", LLMProfile: "careful"},
|
||||||
|
Binding("grounded"),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
extractChain := findResolvedValidatorChain(resolved.ValidatorChains, StageExtract, "events", "event-extractor")
|
||||||
|
if extractChain == nil {
|
||||||
|
t.Fatal("extract validator chain not found")
|
||||||
|
}
|
||||||
|
if len(extractChain.Validators) != 2 {
|
||||||
|
t.Fatalf("extract validators = %#v, want explicit two-validator override", extractChain.Validators)
|
||||||
|
}
|
||||||
|
if extractChain.Validators[0].Binding.Module != "second-validator" || extractChain.Validators[0].Binding.LLMProfile != "careful" {
|
||||||
|
t.Fatalf("first validator = %#v, want explicit LLM-backed validator first", extractChain.Validators[0])
|
||||||
|
}
|
||||||
|
if extractChain.Validators[1].Binding.Module != "grounded" {
|
||||||
|
t.Fatalf("second validator = %#v, want grounded second", extractChain.Validators[1])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineExplicitEmptyValidatorOverrideSuppressesDefaultChain(t *testing.T) {
|
||||||
|
catalog := newProfileCatalog(t)
|
||||||
|
if err := catalog.ValidatorChains.Register(ValidatorChainMapping{
|
||||||
|
Stage: StageExtract,
|
||||||
|
Module: "event-extractor",
|
||||||
|
Validators: []ModuleBinding{Binding("grounded")},
|
||||||
|
}); err != nil {
|
||||||
|
t.Fatalf("register validator chain: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
profile := PipelineProfile{
|
||||||
|
ID: "validated",
|
||||||
|
Input: Binding("text"),
|
||||||
|
Artifacts: map[string]ArtifactLaneProfile{
|
||||||
|
"events": {
|
||||||
|
Extract: ModuleBinding{
|
||||||
|
Module: "event-extractor",
|
||||||
|
Validators: ValidatorOverride{Set: true},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
resolved, err := ResolvePipeline(profile, ResolveOptions{}, catalog)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %v, want nil", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
extractChain := findResolvedValidatorChain(resolved.ValidatorChains, StageExtract, "events", "event-extractor")
|
||||||
|
if extractChain == nil {
|
||||||
|
t.Fatal("extract validator chain not found")
|
||||||
|
}
|
||||||
|
if len(extractChain.Validators) != 0 {
|
||||||
|
t.Fatalf("extract validators = %#v, want explicit empty override", extractChain.Validators)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineRejectsUnknownOverrideValidator(t *testing.T) {
|
||||||
|
_, err := ResolvePipeline(PipelineProfile{
|
||||||
|
ID: "validated",
|
||||||
|
Input: Binding("text"),
|
||||||
|
Artifacts: map[string]ArtifactLaneProfile{
|
||||||
|
"events": {
|
||||||
|
Extract: ModuleBinding{
|
||||||
|
Module: "event-extractor",
|
||||||
|
Validators: ValidatorOverride{
|
||||||
|
Set: true,
|
||||||
|
Validators: []ModuleBinding{Binding("missing-validator")},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}, ResolveOptions{}, newProfileCatalog(t))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("ResolvePipeline() error = nil, want unknown validator error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "missing-validator") {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %q, want missing validator context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestResolvePipelineRejectsLLMProfileForDeterministicValidator(t *testing.T) {
|
||||||
|
_, err := ResolvePipeline(PipelineProfile{
|
||||||
|
ID: "validated",
|
||||||
|
Input: Binding("text"),
|
||||||
|
Artifacts: map[string]ArtifactLaneProfile{
|
||||||
|
"events": {
|
||||||
|
Extract: ModuleBinding{
|
||||||
|
Module: "event-extractor",
|
||||||
|
Validators: ValidatorOverride{
|
||||||
|
Set: true,
|
||||||
|
Validators: []ModuleBinding{
|
||||||
|
{Module: "grounded", LLMProfile: "careful"},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}, ResolveOptions{}, newProfileCatalog(t))
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("ResolvePipeline() error = nil, want deterministic validator profile error")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "grounded") || !strings.Contains(err.Error(), "llm_profile") {
|
||||||
|
t.Fatalf("ResolvePipeline() error = %q, want validator profile context", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestResolvePipelineSelectsOnlyRequestedLanes(t *testing.T) {
|
func TestResolvePipelineSelectsOnlyRequestedLanes(t *testing.T) {
|
||||||
profile := multiLaneProfile()
|
profile := multiLaneProfile()
|
||||||
resolved, err := ResolvePipeline(profile, ResolveOptions{Only: []string{" summaries ", "events", "summaries"}}, newProfileCatalog(t))
|
resolved, err := ResolvePipeline(profile, ResolveOptions{Only: []string{" summaries ", "events", "summaries"}}, newProfileCatalog(t))
|
||||||
@@ -850,7 +987,7 @@ func TestResolvePipelineRejectsConfiguredValidators(t *testing.T) {
|
|||||||
if err == nil {
|
if err == nil {
|
||||||
t.Fatal("ResolvePipeline() error = nil, want error")
|
t.Fatal("ResolvePipeline() error = nil, want error")
|
||||||
}
|
}
|
||||||
assertErrorContains(t, err, "baseline", "events", "configured validators", "not supported")
|
assertErrorContains(t, err, "baseline", "events", "validators", "extract.validators")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestResolvePipelineOrdersLanesDeterministically(t *testing.T) {
|
func TestResolvePipelineOrdersLanesDeterministically(t *testing.T) {
|
||||||
@@ -1139,6 +1276,13 @@ func registerProfileSpecs(t *testing.T, catalog ModuleCatalog, specs ...ModuleSp
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func registerProfileValidatorSpec(t *testing.T, catalog ModuleCatalog, spec ValidatorSpec) {
|
||||||
|
t.Helper()
|
||||||
|
if err := catalog.Validators.RegisterWithSpec(spec, profileValidatorConstructor(spec.Key)); err != nil {
|
||||||
|
t.Fatalf("register validator spec %#v: %v", spec, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func profileInputConstructor(key string) InputAdapterConstructor {
|
func profileInputConstructor(key string) InputAdapterConstructor {
|
||||||
return func() (contracts.InputAdapter, error) {
|
return func() (contracts.InputAdapter, error) {
|
||||||
return profileInputAdapter{key: key}, nil
|
return profileInputAdapter{key: key}, nil
|
||||||
|
|||||||
@@ -631,7 +631,7 @@ func validateRunInput(input RunInput) error {
|
|||||||
return fmt.Errorf("resolved pipeline lane %q normalize module must not be empty", lane.ID)
|
return fmt.Errorf("resolved pipeline lane %q normalize module must not be empty", lane.ID)
|
||||||
}
|
}
|
||||||
if len(lane.Validators) > 0 {
|
if len(lane.Validators) > 0 {
|
||||||
return fmt.Errorf("resolved pipeline lane %q configured validators are not supported by the current raw validation runner", lane.ID)
|
return fmt.Errorf("resolved pipeline lane %q validators are not supported at artifact lane level; use extract.validators, merge.validators, or normalize.validators", lane.ID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
|||||||
@@ -1081,7 +1081,7 @@ func TestRunRejectsConfiguredValidators(t *testing.T) {
|
|||||||
_, err := New(newRunnerRegistries(t, defaultRunnerModules())).Run(context.Background(), RunInput{
|
_, err := New(newRunnerRegistries(t, defaultRunnerModules())).Run(context.Background(), RunInput{
|
||||||
Pipeline: resolvedPipelineWithValidators("configured", "second-validator"),
|
Pipeline: resolvedPipelineWithValidators("configured", "second-validator"),
|
||||||
})
|
})
|
||||||
assertRunError(t, err, "configured validators")
|
assertRunError(t, err, "extract.validators")
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestRunCollectsStageWarnings(t *testing.T) {
|
func TestRunCollectsStageWarnings(t *testing.T) {
|
||||||
|
|||||||
@@ -98,5 +98,13 @@ func cloneModuleBinding(binding ModuleBinding) ModuleBinding {
|
|||||||
}
|
}
|
||||||
binding.References = references
|
binding.References = references
|
||||||
}
|
}
|
||||||
|
binding.Validators = cloneValidatorOverride(binding.Validators)
|
||||||
return binding
|
return binding
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func cloneValidatorOverride(override ValidatorOverride) ValidatorOverride {
|
||||||
|
return ValidatorOverride{
|
||||||
|
Set: override.Set,
|
||||||
|
Validators: cloneModuleBindings(override.Validators),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user