Add validation policy configuration

This commit is contained in:
2026-08-26 23:34:06 +00:00
parent 1c3da3e869
commit 9d0faabf61
14 changed files with 628 additions and 57 deletions

View File

@@ -223,6 +223,7 @@ pipelines:
| --- | --- | --- | --- | | --- | --- | --- | --- |
| **llm_profile** | string | none | Optional non-empty default PromptKit profile ID for selected LLM-backed bindings and validators. An explicitly present blank value is invalid. | | **llm_profile** | string | none | Optional non-empty default PromptKit profile ID for selected LLM-backed bindings and validators. An explicitly present blank value is invalid. |
| **structured_output_repair_attempts** | integer | prompt-owned (1 in maintained production prompts) | Optional structural-repair limit from 0 through 3 for selected LLM-backed bindings and validators. Omission leaves the prompt's declared policy in control; explicit 0 disables structural repair at that scope. | | **structured_output_repair_attempts** | integer | prompt-owned (1 in maintained production prompts) | Optional structural-repair limit from 0 through 3 for selected LLM-backed bindings and validators. Omission leaves the prompt's declared policy in control; explicit 0 disables structural repair at that scope. |
| **validation_policy** | object | see below | Optional terminal policy defaults for producer validation. Its fields inherit independently into chunk, extract, merge, and normalize bindings. |
| **input** | module binding | none | Required. | | **input** | module binding | none | Required. |
| **chunk** | module binding | **generic** | Optional. | | **chunk** | module binding | **generic** | Optional. |
| **output** | module binding | **json** | Optional. | | **output** | module binding | **json** | Optional. |
@@ -251,6 +252,24 @@ explicit `null` and non-integer values are invalid. An explicit value on a
deterministic binding or validator is invalid, while a pipeline value simply deterministic binding or validator is invalid, while a pipeline value simply
does not apply to deterministic selections. does not apply to deterministic selections.
`validation_policy` records terminal-policy metadata for a complete producer
attempt and validation chain. It may appear on a pipeline or a **chunk**,
**extract**, **merge**, or **normalize** module binding; input, output, and
validator bindings reject it. Every field is optional and resolves in binding,
pipeline, then application-default order. The current runner records the
resolved values but does not yet consume them for terminal disposition:
| Field | Values | Default |
| --- | --- | --- |
| **producer_structural_failure** | **fail_run**, **reject_output** | **fail_run** |
| **semantic_rejection** | **fail_run**, **reject_output** | **fail_run** |
| **validator_failure** | **warn_continue**, **fail_run** | **warn_continue** |
The policy object and its fields must be non-null, and unknown fields are
rejected. A deterministic producer may not explicitly set
**producer_structural_failure** on its binding, although a pipeline-level
default remains valid for pipelines that include LLM-backed producers.
A lane has these fields: A lane has these fields:
| Field | Type | Default | Rules | | Field | Type | Default | Rules |
@@ -289,6 +308,7 @@ extract:
| **module** | string | none | Required for an object binding. Must be a registered compatible key. | | **module** | string | none | Required for an object binding. Must be a registered compatible key. |
| **llm_profile** | string | none | Optional non-empty PromptKit profile ID for an LLM-backed binding. It overrides the pipeline default unless the run supplies **--llm-profile**. | | **llm_profile** | string | none | Optional non-empty PromptKit profile ID for an LLM-backed binding. It overrides the pipeline default unless the run supplies **--llm-profile**. |
| **structured_output_repair_attempts** | integer | pipeline or prompt-owned (1 in maintained production prompts) | Optional structural-repair limit from 0 through 3 for an LLM-backed binding. It overrides the pipeline value; explicit 0 disables structural repair. | | **structured_output_repair_attempts** | integer | pipeline or prompt-owned (1 in maintained production prompts) | Optional structural-repair limit from 0 through 3 for an LLM-backed binding. It overrides the pipeline value; explicit 0 disables structural repair. |
| **validation_policy** | object | pipeline or application defaults | Optional field-by-field terminal-policy override for a chunk, extract, merge, or normalize binding. |
| **retries** | integer | 0 | Non-negative additional attempts for chunk, extract, merge, and normalize bindings. | | **retries** | integer | 0 | Non-negative additional attempts for chunk, extract, merge, and normalize bindings. |
| **options** | object | none | Must satisfy the selected module. | | **options** | object | none | Must satisfy the selected module. |
| **references** | map | none | Valid only on chunk, extract, merge, and normalize bindings. | | **references** | map | none | Valid only on chunk, extract, merge, and normalize bindings. |
@@ -298,6 +318,7 @@ Omitting **validators** uses the registered chain. **validators: []** selects
an empty chain; a non-empty list replaces the chain in the listed order. an empty chain; a non-empty list replaces the chain in the listed order.
Validator bindings accept only **module**, **llm_profile**, Validator bindings accept only **module**, **llm_profile**,
**structured_output_repair_attempts**, and **options**. They reject **structured_output_repair_attempts**, and **options**. They reject
**validation_policy**,
**references**, **retries**, and nested **validators**. Deterministic validators **references**, **retries**, and nested **validators**. Deterministic validators
reject explicit **llm_profile** and **structured_output_repair_attempts**. reject explicit **llm_profile** and **structured_output_repair_attempts**.
Deterministic module bindings also reject those explicit fields. Deterministic module bindings also reject those explicit fields.

View File

@@ -74,6 +74,12 @@ profile-free, and no second inheritance decision occurs during execution. The
public field definitions and precedence are owned by public field definitions and precedence are owned by
[Configuration](../config.md#pipelines). [Configuration](../config.md#pipelines).
The resolver retains configured `validation_policy` overrides and derives one
detached concrete terminal policy for the chunk producer and every lane's
extract, merge, and normalize producers. That field-by-field inheritance is
complete before preparation, and the effective values contribute to pipeline
and checkpoint identity; execution does not interpret configuration defaults.
The framework resolver supplies defaults, selects lanes, resolves validator The framework resolver supplies defaults, selects lanes, resolves validator
chains, checks registered module and artifact compatibility, validates module chains, checks registered module and artifact compatibility, validates module
options, and returns the fixed ordered pipeline shape. The resulting options, and returns the fixed ordered pipeline shape. The resulting

View File

@@ -189,7 +189,7 @@ The generic contracts can safely carry correction input and exact candidate
material across every producer stage, no PromptKit type crosses the boundary, material across every producer stage, no PromptKit type crosses the boundary,
and existing runtime behavior is unchanged. This stage is one Terra prompt. and existing runtime behavior is unchanged. This stage is one Terra prompt.
## Stage 3 — Add Validation Policy Configuration And Resolution ## Stage 3 — Add Validation Policy Configuration And Resolution
### Goal ### Goal

View File

@@ -116,6 +116,7 @@ func (c *ConcurrencyConfig) recomputeStageWorkerDefaults() {
func clonePipelineProfile(in pipeline.PipelineProfile) pipeline.PipelineProfile { func clonePipelineProfile(in pipeline.PipelineProfile) pipeline.PipelineProfile {
out := in out := in
out.ValidationPolicy = cloneValidationPolicyOverride(in.ValidationPolicy)
if in.StructuredOutputRepairAttempts != nil { if in.StructuredOutputRepairAttempts != nil {
value := *in.StructuredOutputRepairAttempts value := *in.StructuredOutputRepairAttempts
out.StructuredOutputRepairAttempts = &value out.StructuredOutputRepairAttempts = &value
@@ -200,6 +201,7 @@ func cloneReferenceSource(in pipeline.ReferenceSource) pipeline.ReferenceSource
func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding { func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
out := in out := in
out.ValidationPolicy = cloneValidationPolicyOverride(in.ValidationPolicy)
if in.StructuredOutputRepairAttempts != nil { if in.StructuredOutputRepairAttempts != nil {
value := *in.StructuredOutputRepairAttempts value := *in.StructuredOutputRepairAttempts
out.StructuredOutputRepairAttempts = &value out.StructuredOutputRepairAttempts = &value
@@ -212,6 +214,26 @@ func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
return out return out
} }
func cloneValidationPolicyOverride(in *pipeline.ValidationPolicyOverride) *pipeline.ValidationPolicyOverride {
if in == nil {
return nil
}
out := *in
if in.ProducerStructuralFailure != nil {
value := *in.ProducerStructuralFailure
out.ProducerStructuralFailure = &value
}
if in.SemanticRejection != nil {
value := *in.SemanticRejection
out.SemanticRejection = &value
}
if in.ValidatorFailure != nil {
value := *in.ValidatorFailure
out.ValidatorFailure = &value
}
return &out
}
func cloneValidatorOverride(in pipeline.ValidatorOverride) pipeline.ValidatorOverride { func cloneValidatorOverride(in pipeline.ValidatorOverride) pipeline.ValidatorOverride {
out := pipeline.ValidatorOverride{Set: in.Set} out := pipeline.ValidatorOverride{Set: in.Set}
if len(in.Validators) > 0 { if len(in.Validators) > 0 {

View File

@@ -37,6 +37,7 @@ type FilePromptKitLocalBackendConfig struct {
type FilePipelineProfile struct { type FilePipelineProfile struct {
LLMProfile *string `yaml:"llm_profile,omitempty"` LLMProfile *string `yaml:"llm_profile,omitempty"`
StructuredOutputRepairAttempts *int `yaml:"structured_output_repair_attempts,omitempty"` StructuredOutputRepairAttempts *int `yaml:"structured_output_repair_attempts,omitempty"`
ValidationPolicy *pipeline.ValidationPolicyOverride `yaml:"validation_policy,omitempty"`
Input fileModuleBinding `yaml:"input"` Input fileModuleBinding `yaml:"input"`
Chunk *fileModuleBinding `yaml:"chunk,omitempty"` Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"` Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
@@ -55,12 +56,19 @@ func (p *FilePipelineProfile) UnmarshalYAML(node *yaml.Node) error {
type plainFilePipelineProfile FilePipelineProfile type plainFilePipelineProfile FilePipelineProfile
var decoded plainFilePipelineProfile var decoded plainFilePipelineProfile
seen, err := decodeKnownMapping(node, &decoded, map[string]struct{}{ seen, err := decodeKnownMapping(node, &decoded, map[string]struct{}{
"llm_profile": {}, "structured_output_repair_attempts": {}, "input": {}, "chunk": {}, "artifacts": {}, "steps": {}, "output": {}, "references": {}, "llm_profile": {}, "structured_output_repair_attempts": {}, "validation_policy": {}, "input": {}, "chunk": {}, "artifacts": {}, "steps": {}, "output": {}, "references": {},
}, "pipeline profile") }, "pipeline profile")
if err != nil { if err != nil {
return err return err
} }
*p = FilePipelineProfile(decoded) *p = FilePipelineProfile(decoded)
if validationPolicyNode, ok := mappingValue(node, "validation_policy"); ok {
policy, err := parseValidationPolicy(validationPolicyNode, "pipeline profile")
if err != nil {
return err
}
p.ValidationPolicy = policy
}
_, p.artifactsSet = seen["artifacts"] _, p.artifactsSet = seen["artifacts"]
_, p.stepsSet = seen["steps"] _, p.stepsSet = seen["steps"]
_, p.llmProfileSet = seen["llm_profile"] _, p.llmProfileSet = seen["llm_profile"]
@@ -154,6 +162,7 @@ type fileModuleBinding struct {
Module string Module string
LLMProfile string LLMProfile string
StructuredOutputRepairAttempts *int StructuredOutputRepairAttempts *int
ValidationPolicy *pipeline.ValidationPolicyOverride
Retries int Retries int
Options map[string]any Options map[string]any
References map[string]fileReferenceSource References map[string]fileReferenceSource
@@ -250,9 +259,14 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
b.Module = strings.TrimSpace(module) b.Module = strings.TrimSpace(module)
return nil return nil
case yaml.MappingNode: case yaml.MappingNode:
seen := make(map[string]struct{}, len(node.Content)/2)
for i := 0; i < len(node.Content); i += 2 { for i := 0; i < len(node.Content); i += 2 {
keyNode := node.Content[i] keyNode := node.Content[i]
valueNode := node.Content[i+1] valueNode := node.Content[i+1]
if _, exists := seen[keyNode.Value]; exists {
return fmt.Errorf("module binding field %q is duplicated", keyNode.Value)
}
seen[keyNode.Value] = struct{}{}
switch keyNode.Value { switch keyNode.Value {
case "module": case "module":
var module string var module string
@@ -275,6 +289,12 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
return err return err
} }
b.StructuredOutputRepairAttempts = attempts b.StructuredOutputRepairAttempts = attempts
case "validation_policy":
policy, err := parseValidationPolicy(valueNode, "module binding")
if err != nil {
return err
}
b.ValidationPolicy = policy
case "retries": case "retries":
var retries int var retries int
if err := valueNode.Decode(&retries); err != nil { if err := valueNode.Decode(&retries); err != nil {
@@ -318,6 +338,7 @@ func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
Module: strings.TrimSpace(b.Module), Module: strings.TrimSpace(b.Module),
LLMProfile: strings.TrimSpace(b.LLMProfile), LLMProfile: strings.TrimSpace(b.LLMProfile),
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(b.StructuredOutputRepairAttempts), StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(b.StructuredOutputRepairAttempts),
ValidationPolicy: cloneValidationPolicyOverride(b.ValidationPolicy),
Retries: b.Retries, Retries: b.Retries,
Options: cloneOptions(b.Options), Options: cloneOptions(b.Options),
References: fileReferenceSourcesToPipeline(b.References), References: fileReferenceSourcesToPipeline(b.References),
@@ -325,6 +346,51 @@ func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
} }
} }
func mappingValue(node *yaml.Node, key string) (*yaml.Node, bool) {
for i := 0; i < len(node.Content); i += 2 {
if node.Content[i].Value == key {
return node.Content[i+1], true
}
}
return nil, false
}
func parseValidationPolicy(node *yaml.Node, context string) (*pipeline.ValidationPolicyOverride, error) {
if node == nil || node.Tag == "!!null" || node.Kind != yaml.MappingNode {
return nil, fmt.Errorf("%s validation_policy must be an object", context)
}
policy := &pipeline.ValidationPolicyOverride{}
seen := make(map[string]struct{}, len(node.Content)/2)
for i := 0; i < len(node.Content); i += 2 {
key := node.Content[i].Value
value := node.Content[i+1]
if _, exists := seen[key]; exists {
return nil, fmt.Errorf("%s validation_policy field %q is duplicated", context, key)
}
seen[key] = struct{}{}
if value.Tag == "!!null" || value.Kind != yaml.ScalarNode || value.Tag != "!!str" {
return nil, fmt.Errorf("%s validation_policy.%s must be a string", context, key)
}
switch key {
case "producer_structural_failure":
value := pipeline.ProducerStructuralFailureAction(value.Value)
policy.ProducerStructuralFailure = &value
case "semantic_rejection":
value := pipeline.SemanticRejectionAction(value.Value)
policy.SemanticRejection = &value
case "validator_failure":
value := pipeline.ValidatorFailureAction(value.Value)
policy.ValidatorFailure = &value
default:
return nil, fmt.Errorf("field %s not found in %s validation_policy", key, context)
}
}
if err := policy.Validate(); err != nil {
return nil, fmt.Errorf("%s validation_policy: %w", context, err)
}
return policy, nil
}
func validateStructuredOutputRepairAttemptsNode(node *yaml.Node, context string) error { func validateStructuredOutputRepairAttemptsNode(node *yaml.Node, context string) error {
if node.Kind != yaml.MappingNode { if node.Kind != yaml.MappingNode {
return fmt.Errorf("%s must be an object", context) return fmt.Errorf("%s must be an object", context)
@@ -573,6 +639,7 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
ID: pipelineID, ID: pipelineID,
LLMProfile: llmProfile, LLMProfile: llmProfile,
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(filePipeline.StructuredOutputRepairAttempts), StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(filePipeline.StructuredOutputRepairAttempts),
ValidationPolicy: cloneValidationPolicyOverride(filePipeline.ValidationPolicy),
Input: filePipeline.Input.toPipelineBinding(), Input: filePipeline.Input.toPipelineBinding(),
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)), Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
References: fileReferenceSourcesToPipeline(filePipeline.References), References: fileReferenceSourcesToPipeline(filePipeline.References),

View File

@@ -32,6 +32,7 @@ func (e EffectiveConfig) RedactedResolvedPipelinePayload() pipeline.ResolvedPipe
func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeline { func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeline {
out := in out := in
out.ConfiguredValidationPolicy = cloneValidationPolicyOverride(in.ConfiguredValidationPolicy)
out.Input = redactBinding(cloneModuleBinding(in.Input)) out.Input = redactBinding(cloneModuleBinding(in.Input))
out.Chunk = redactBinding(cloneModuleBinding(in.Chunk)) out.Chunk = redactBinding(cloneModuleBinding(in.Chunk))
out.ChunkReferences = pipeline.CloneReferenceTarget(in.ChunkReferences) out.ChunkReferences = pipeline.CloneReferenceTarget(in.ChunkReferences)

View File

@@ -225,6 +225,39 @@ func TestRedactedResolvedPipelinePayloadHandlesTypedOptionContainers(t *testing.
} }
} }
func TestRedactedEffectiveConfigPayloadOwnsValidationPolicies(t *testing.T) {
semantic := pipeline.SemanticRejectionRejectOutput
validator := pipeline.ValidatorFailureFailRun
configured := &pipeline.ValidationPolicyOverride{SemanticRejection: &semantic, ValidatorFailure: &validator}
effective := EffectiveConfig{
Config: Config{Pipelines: map[string]pipeline.PipelineProfile{
"main": {ValidationPolicy: configured},
}},
ResolvedPipeline: pipeline.ResolvedPipeline{
ConfiguredValidationPolicy: configured,
ChunkValidationPolicy: pipeline.ValidationPolicy{
ProducerStructuralFailure: pipeline.ProducerStructuralFailureFailRun,
SemanticRejection: pipeline.SemanticRejectionRejectOutput,
ValidatorFailure: pipeline.ValidatorFailureFailRun,
},
},
}
payload := effective.RedactedSummaryPayload().(EffectiveConfig)
encoded, err := json.Marshal(payload)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(encoded), `"configured_validation_policy":{"semantic_rejection":"reject_output","validator_failure":"fail_run"}`) || !strings.Contains(string(encoded), `"chunk_validation_policy":{"producer_structural_failure":"fail_run","semantic_rejection":"reject_output","validator_failure":"fail_run"}`) {
t.Fatalf("redacted payload omitted validation policy: %s", encoded)
}
*payload.Config.Pipelines["main"].ValidationPolicy.SemanticRejection = pipeline.SemanticRejectionFailRun
*payload.ResolvedPipeline.ConfiguredValidationPolicy.ValidatorFailure = pipeline.ValidatorFailureWarnContinue
if *effective.Config.Pipelines["main"].ValidationPolicy.SemanticRejection != pipeline.SemanticRejectionRejectOutput || *effective.ResolvedPipeline.ConfiguredValidationPolicy.ValidatorFailure != pipeline.ValidatorFailureFailRun {
t.Fatal("redacted payload aliases validation policy")
}
}
func redactionTestBinding(name string) pipeline.ModuleBinding { func redactionTestBinding(name string) pipeline.ModuleBinding {
return pipeline.ModuleBinding{ return pipeline.ModuleBinding{
Module: "safe-" + name, Module: "safe-" + name,

View File

@@ -119,6 +119,11 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile) erro
if err := validateStructuredOutputRepairAttempts(fmt.Sprintf("pipeline %q", id), profile.StructuredOutputRepairAttempts); err != nil { if err := validateStructuredOutputRepairAttempts(fmt.Sprintf("pipeline %q", id), profile.StructuredOutputRepairAttempts); err != nil {
return err return err
} }
if profile.ValidationPolicy != nil {
if err := profile.ValidationPolicy.Validate(); err != nil {
return fmt.Errorf("pipeline %q validation_policy: %w", id, err)
}
}
if err := validateBinding(id, "", "input", profile.Input, false); err != nil { if err := validateBinding(id, "", "input", profile.Input, false); err != nil {
return err return err
} }
@@ -198,6 +203,16 @@ func validateBinding(
binding pipeline.ModuleBinding, binding pipeline.ModuleBinding,
referencesAllowed bool, referencesAllowed bool,
) error { ) error {
if binding.ValidationPolicy != nil {
switch slot {
case "chunk", "extract", "merge", "normalize":
default:
return fmt.Errorf("%s validation_policy is not supported", referenceContext(pipelineID, laneID, slot))
}
if err := binding.ValidationPolicy.Validate(); err != nil {
return fmt.Errorf("%s validation_policy: %w", referenceContext(pipelineID, laneID, slot), err)
}
}
if err := validateStructuredOutputRepairAttempts(referenceContext(pipelineID, laneID, slot), binding.StructuredOutputRepairAttempts); err != nil { if err := validateStructuredOutputRepairAttempts(referenceContext(pipelineID, laneID, slot), binding.StructuredOutputRepairAttempts); err != nil {
return err return err
} }
@@ -255,6 +270,9 @@ func validateValidatorOverride(pipelineID string, laneID string, slot string, ov
if validator.Validators.Set { if validator.Validators.Set {
return fmt.Errorf("%s nested validators are not supported", context) return fmt.Errorf("%s nested validators are not supported", context)
} }
if validator.ValidationPolicy != nil {
return fmt.Errorf("%s validation_policy is not supported", context)
}
if validator.Retries != 0 { if validator.Retries != 0 {
return fmt.Errorf("%s retries are not supported", context) return fmt.Errorf("%s retries are not supported", context)
} }

View File

@@ -0,0 +1,97 @@
package config
import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestValidationPolicyFileConfigurationIsStrictAndPresenceAware(t *testing.T) {
const valid = `version: 4
pipelines:
main:
validation_policy:
producer_structural_failure: reject_output
semantic_rejection: fail_run
input: seriatim
chunk:
module: generic
validation_policy:
validator_failure: fail_run
artifacts:
lane:
extract:
module: extract
validation_policy:
semantic_rejection: reject_output
`
cfg := applyFileConfig(t, valid)
profile := cfg.Pipelines["main"]
if profile.ValidationPolicy == nil || profile.ValidationPolicy.ProducerStructuralFailure == nil || *profile.ValidationPolicy.ProducerStructuralFailure != pipeline.ProducerStructuralFailureRejectOutput || profile.ValidationPolicy.SemanticRejection == nil || *profile.ValidationPolicy.SemanticRejection != pipeline.SemanticRejectionFailRun || profile.ValidationPolicy.ValidatorFailure != nil {
t.Fatalf("pipeline validation policy = %#v", profile.ValidationPolicy)
}
if profile.Chunk.ValidationPolicy == nil || profile.Chunk.ValidationPolicy.ValidatorFailure == nil || *profile.Chunk.ValidationPolicy.ValidatorFailure != pipeline.ValidatorFailureFailRun {
t.Fatalf("chunk validation policy = %#v", profile.Chunk.ValidationPolicy)
}
lane := profile.Artifacts["lane"]
if lane.Extract.ValidationPolicy == nil || lane.Extract.ValidationPolicy.SemanticRejection == nil || *lane.Extract.ValidationPolicy.SemanticRejection != pipeline.SemanticRejectionRejectOutput {
t.Fatalf("extract validation policy = %#v", lane.Extract.ValidationPolicy)
}
for _, test := range []struct {
name string
yaml string
}{
{"null object", strings.Replace(valid, "validation_policy:\n producer_structural_failure: reject_output\n semantic_rejection: fail_run", "validation_policy: null", 1)},
{"null field", strings.Replace(valid, "semantic_rejection: fail_run", "semantic_rejection: null", 1)},
{"unknown field", strings.Replace(valid, "semantic_rejection: fail_run", "unknown: fail_run", 1)},
{"duplicate field", strings.Replace(valid, "semantic_rejection: fail_run", "semantic_rejection: fail_run\n semantic_rejection: reject_output", 1)},
{"invalid enum", strings.Replace(valid, "semantic_rejection: fail_run", "semantic_rejection: continue", 1)},
} {
t.Run(test.name, func(t *testing.T) {
if _, err := ParseFileConfigYAML([]byte(test.yaml)); err == nil {
t.Fatal("ParseFileConfigYAML() error = nil, want strict validation-policy rejection")
}
})
}
}
func TestValidationPolicyPlacementRules(t *testing.T) {
policy := &pipeline.ValidationPolicyOverride{}
semantic := pipeline.SemanticRejectionRejectOutput
policy.SemanticRejection = &semantic
base := pipeline.PipelineProfile{
ID: "main",
Input: pipeline.Binding("input"),
Artifacts: map[string]pipeline.ArtifactLaneProfile{
"lane": {Extract: pipeline.Binding("extract")},
},
}
for _, test := range []struct {
name string
mutate func(*pipeline.PipelineProfile)
}{
{"input", func(profile *pipeline.PipelineProfile) { profile.Input.ValidationPolicy = policy }},
{"output", func(profile *pipeline.PipelineProfile) {
profile.Output = pipeline.Binding("output")
profile.Output.ValidationPolicy = policy
}},
{"validator", func(profile *pipeline.PipelineProfile) {
lane := profile.Artifacts["lane"]
lane.Extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{{Module: "validator", ValidationPolicy: policy}}}
profile.Artifacts["lane"] = lane
}},
} {
t.Run(test.name, func(t *testing.T) {
profile := base
profile.Artifacts = map[string]pipeline.ArtifactLaneProfile{"lane": base.Artifacts["lane"]}
test.mutate(&profile)
cfg := Default()
cfg.Pipelines = map[string]pipeline.PipelineProfile{"main": profile}
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "validation_policy") {
t.Fatalf("Config.Validate() error = %v, want placement rejection", err)
}
})
}
}

View File

@@ -406,6 +406,7 @@ func validateRegistrySet(resolved ResolvedPipeline, registries Registries) error
func cloneResolvedPipeline(in ResolvedPipeline) ResolvedPipeline { func cloneResolvedPipeline(in ResolvedPipeline) ResolvedPipeline {
out := in out := in
out.ConfiguredValidationPolicy = cloneValidationPolicyOverride(in.ConfiguredValidationPolicy)
out.Input = cloneModuleBinding(in.Input) out.Input = cloneModuleBinding(in.Input)
out.Chunk = cloneModuleBinding(in.Chunk) out.Chunk = cloneModuleBinding(in.Chunk)
out.Output = cloneModuleBinding(in.Output) out.Output = cloneModuleBinding(in.Output)

View File

@@ -24,6 +24,7 @@ type ModuleBinding struct {
Module string `json:"module"` Module string `json:"module"`
LLMProfile string `json:"llm_profile,omitempty"` LLMProfile string `json:"llm_profile,omitempty"`
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"` StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
ValidationPolicy *ValidationPolicyOverride `json:"validation_policy,omitempty"`
Retries int `json:"retries,omitempty"` Retries int `json:"retries,omitempty"`
Options map[string]any `json:"options,omitempty"` Options map[string]any `json:"options,omitempty"`
References map[string]ReferenceSource `json:"references,omitempty"` References map[string]ReferenceSource `json:"references,omitempty"`
@@ -84,6 +85,7 @@ func (binding ModuleBinding) MarshalJSON() ([]byte, error) {
Module string `json:"module"` Module string `json:"module"`
LLMProfile string `json:"llm_profile,omitempty"` LLMProfile string `json:"llm_profile,omitempty"`
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"` StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
ValidationPolicy *ValidationPolicyOverride `json:"validation_policy,omitempty"`
Retries int `json:"retries,omitempty"` Retries int `json:"retries,omitempty"`
Options map[string]any `json:"options,omitempty"` Options map[string]any `json:"options,omitempty"`
References map[string]ReferenceSource `json:"references,omitempty"` References map[string]ReferenceSource `json:"references,omitempty"`
@@ -93,6 +95,7 @@ func (binding ModuleBinding) MarshalJSON() ([]byte, error) {
Module: binding.Module, Module: binding.Module,
LLMProfile: binding.LLMProfile, LLMProfile: binding.LLMProfile,
StructuredOutputRepairAttempts: binding.StructuredOutputRepairAttempts, StructuredOutputRepairAttempts: binding.StructuredOutputRepairAttempts,
ValidationPolicy: cloneValidationPolicyOverride(binding.ValidationPolicy),
Retries: binding.Retries, Retries: binding.Retries,
Options: binding.Options, Options: binding.Options,
References: binding.References, References: binding.References,
@@ -112,6 +115,7 @@ func (binding *ModuleBinding) UnmarshalJSON(data []byte) error {
Module string `json:"module"` Module string `json:"module"`
LLMProfile string `json:"llm_profile,omitempty"` LLMProfile string `json:"llm_profile,omitempty"`
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"` StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
ValidationPolicy *ValidationPolicyOverride `json:"validation_policy,omitempty"`
Retries int `json:"retries,omitempty"` Retries int `json:"retries,omitempty"`
Options map[string]any `json:"options,omitempty"` Options map[string]any `json:"options,omitempty"`
References map[string]ReferenceSource `json:"references,omitempty"` References map[string]ReferenceSource `json:"references,omitempty"`
@@ -125,6 +129,7 @@ func (binding *ModuleBinding) UnmarshalJSON(data []byte) error {
Module: decoded.Module, Module: decoded.Module,
LLMProfile: decoded.LLMProfile, LLMProfile: decoded.LLMProfile,
StructuredOutputRepairAttempts: decoded.StructuredOutputRepairAttempts, StructuredOutputRepairAttempts: decoded.StructuredOutputRepairAttempts,
ValidationPolicy: cloneValidationPolicyOverride(decoded.ValidationPolicy),
Retries: decoded.Retries, Retries: decoded.Retries,
Options: decoded.Options, Options: decoded.Options,
References: decoded.References, References: decoded.References,
@@ -158,6 +163,7 @@ type PipelineProfile struct {
ID string `json:"id"` ID string `json:"id"`
LLMProfile string `json:"llm_profile,omitempty"` LLMProfile string `json:"llm_profile,omitempty"`
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"` StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
ValidationPolicy *ValidationPolicyOverride `json:"validation_policy,omitempty"`
Input ModuleBinding `json:"input"` Input ModuleBinding `json:"input"`
Chunk ModuleBinding `json:"chunk,omitempty"` Chunk ModuleBinding `json:"chunk,omitempty"`
Artifacts map[string]ArtifactLaneProfile `json:"artifacts"` Artifacts map[string]ArtifactLaneProfile `json:"artifacts"`
@@ -198,23 +204,26 @@ type ResolvedReferenceTarget struct {
} }
type ResolvedArtifactLane struct { type ResolvedArtifactLane struct {
StepID string StepID string
ID string ID string
ArtifactKind contracts.ArtifactKind `json:"artifact_kind,omitempty"` ArtifactKind contracts.ArtifactKind `json:"artifact_kind,omitempty"`
ArtifactSchemaID string `json:"artifact_schema_id,omitempty"` ArtifactSchemaID string `json:"artifact_schema_id,omitempty"`
ArtifactSchemaName string `json:"artifact_schema_name,omitempty"` ArtifactSchemaName string `json:"artifact_schema_name,omitempty"`
ArtifactSchemaVersion string `json:"artifact_schema_version,omitempty"` ArtifactSchemaVersion string `json:"artifact_schema_version,omitempty"`
ArtifactSchemaDigest string `json:"artifact_schema_digest,omitempty"` ArtifactSchemaDigest string `json:"artifact_schema_digest,omitempty"`
Extract ModuleBinding Extract ModuleBinding
ExtractExecutionClass contracts.ExecutionClass `json:"extract_execution_class"` ExtractExecutionClass contracts.ExecutionClass `json:"extract_execution_class"`
Merge ModuleBinding ExtractValidationPolicy ValidationPolicy `json:"extract_validation_policy"`
MergeExecutionClass contracts.ExecutionClass `json:"merge_execution_class"` Merge ModuleBinding
Normalize ModuleBinding MergeExecutionClass contracts.ExecutionClass `json:"merge_execution_class"`
NormalizeExecutionClass contracts.ExecutionClass `json:"normalize_execution_class"` MergeValidationPolicy ValidationPolicy `json:"merge_validation_policy"`
Validators []ModuleBinding Normalize ModuleBinding
ExtractReferences ResolvedReferenceTarget `json:"extract_references"` NormalizeExecutionClass contracts.ExecutionClass `json:"normalize_execution_class"`
MergeReferences ResolvedReferenceTarget `json:"merge_references"` NormalizeValidationPolicy ValidationPolicy `json:"normalize_validation_policy"`
NormalizeReferences ResolvedReferenceTarget `json:"normalize_references"` Validators []ModuleBinding
ExtractReferences ResolvedReferenceTarget `json:"extract_references"`
MergeReferences ResolvedReferenceTarget `json:"merge_references"`
NormalizeReferences ResolvedReferenceTarget `json:"normalize_references"`
} }
type ResolvedPipelineStep struct { type ResolvedPipelineStep struct {
@@ -237,17 +246,19 @@ type ResolvedValidator struct {
} }
type ResolvedPipeline struct { type ResolvedPipeline struct {
ID string ID string
Digest string Digest string
Input ModuleBinding ConfiguredValidationPolicy *ValidationPolicyOverride `json:"configured_validation_policy,omitempty"`
InputExecutionClass contracts.ExecutionClass `json:"input_execution_class"` Input ModuleBinding
Chunk ModuleBinding InputExecutionClass contracts.ExecutionClass `json:"input_execution_class"`
ChunkExecutionClass contracts.ExecutionClass `json:"chunk_execution_class"` Chunk ModuleBinding
ChunkReferences ResolvedReferenceTarget `json:"chunk_references"` ChunkExecutionClass contracts.ExecutionClass `json:"chunk_execution_class"`
Steps []ResolvedPipelineStep ChunkValidationPolicy ValidationPolicy `json:"chunk_validation_policy"`
ValidatorChains []ResolvedValidatorChain `json:"validator_chains"` ChunkReferences ResolvedReferenceTarget `json:"chunk_references"`
Output ModuleBinding Steps []ResolvedPipelineStep
OutputExecutionClass contracts.ExecutionClass `json:"output_execution_class"` ValidatorChains []ResolvedValidatorChain `json:"validator_chains"`
Output ModuleBinding
OutputExecutionClass contracts.ExecutionClass `json:"output_execution_class"`
} }
// AllArtifactLanes returns lanes in deterministic step order for read-only // AllArtifactLanes returns lanes in deterministic step order for read-only
@@ -414,13 +425,14 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
return ResolvedPipeline{}, err return ResolvedPipeline{}, err
} }
resolved := ResolvedPipeline{ resolved := ResolvedPipeline{
ID: pipelineID, ID: pipelineID,
Input: input, ConfiguredValidationPolicy: cloneValidationPolicyOverride(profile.ValidationPolicy),
InputExecutionClass: inputModuleSpec.ExecutionClass, Input: input,
Chunk: chunk, InputExecutionClass: inputModuleSpec.ExecutionClass,
ChunkExecutionClass: chunkSpec.ExecutionClass, Chunk: chunk,
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences), ChunkExecutionClass: chunkSpec.ExecutionClass,
Output: output, ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences),
Output: output,
} }
chunkValidatorChain, err := resolveValidatorChain(pipelineID, "", StageChunk, chunk.Module, chunk.Validators, "", nil, catalog) chunkValidatorChain, err := resolveValidatorChain(pipelineID, "", StageChunk, chunk.Module, chunk.Validators, "", nil, catalog)
if err != nil { if err != nil {
@@ -493,6 +505,9 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
if err := applyEffectiveStructuredOutputRepairAttempts(&resolved, profile.StructuredOutputRepairAttempts); err != nil { if err := applyEffectiveStructuredOutputRepairAttempts(&resolved, profile.StructuredOutputRepairAttempts); err != nil {
return ResolvedPipeline{}, err return ResolvedPipeline{}, err
} }
if err := applyEffectiveValidationPolicies(&resolved, profile.ValidationPolicy); err != nil {
return ResolvedPipeline{}, err
}
if err := validateResolvedOptions(resolved, catalog, configuredLaneIDs); err != nil { if err := validateResolvedOptions(resolved, catalog, configuredLaneIDs); err != nil {
return ResolvedPipeline{}, err return ResolvedPipeline{}, err
} }
@@ -1328,6 +1343,7 @@ func resolveBinding(binding ModuleBinding, defaultModule string, referenceSlotLa
Module: module, Module: module,
LLMProfile: llmProfile, LLMProfile: llmProfile,
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts), StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts),
ValidationPolicy: cloneValidationPolicyOverride(binding.ValidationPolicy),
Retries: binding.Retries, Retries: binding.Retries,
Options: cloneOptions(binding.Options), Options: cloneOptions(binding.Options),
References: references, References: references,
@@ -1456,6 +1472,72 @@ func applyEffectiveStructuredOutputRepairAttempts(resolved *ResolvedPipeline, pi
return nil return nil
} }
func applyEffectiveValidationPolicies(resolved *ResolvedPipeline, pipelinePolicy *ValidationPolicyOverride) error {
apply := func(stage ModuleStage, laneID string, binding ModuleBinding, executionClass contracts.ExecutionClass) (ValidationPolicy, error) {
binding.ValidationPolicy = cloneValidationPolicyOverride(binding.ValidationPolicy)
if binding.ValidationPolicy != nil {
if err := binding.ValidationPolicy.Validate(); err != nil {
return ValidationPolicy{}, fmt.Errorf("pipeline %q %s validation_policy: %w", resolved.ID, stage, err)
}
if executionClass == contracts.ExecutionClassDeterministic && binding.ValidationPolicy.ProducerStructuralFailure != nil {
if laneID == "" {
return ValidationPolicy{}, fmt.Errorf("pipeline %q %s %q assigns producer_structural_failure to deterministic module", resolved.ID, stage, binding.Module)
}
return ValidationPolicy{}, fmt.Errorf("pipeline %q lane %q %s %q assigns producer_structural_failure to deterministic module", resolved.ID, laneID, stage, binding.Module)
}
}
return ResolveValidationPolicy(binding.ValidationPolicy, pipelinePolicy), nil
}
if pipelinePolicy != nil {
if err := pipelinePolicy.Validate(); err != nil {
return fmt.Errorf("pipeline %q validation_policy: %w", resolved.ID, err)
}
}
if resolved.Input.ValidationPolicy != nil {
return fmt.Errorf("pipeline %q input validation_policy is not supported", resolved.ID)
}
if resolved.Output.ValidationPolicy != nil {
return fmt.Errorf("pipeline %q output validation_policy is not supported", resolved.ID)
}
for _, chain := range resolved.ValidatorChains {
for _, validator := range chain.Validators {
if validator.Binding.ValidationPolicy != nil {
if chain.LaneID == "" {
return fmt.Errorf("pipeline %q %s validator %q validation_policy is not supported", resolved.ID, chain.Stage, validator.Binding.Module)
}
return fmt.Errorf("pipeline %q lane %q %s validator %q validation_policy is not supported", resolved.ID, chain.LaneID, chain.Stage, validator.Binding.Module)
}
}
}
policy, err := apply(StageChunk, "", resolved.Chunk, resolved.ChunkExecutionClass)
if err != nil {
return err
}
resolved.ChunkValidationPolicy = policy
for stepIndex := range resolved.Steps {
for laneIndex := range resolved.Steps[stepIndex].ArtifactLanes {
lane := &resolved.Steps[stepIndex].ArtifactLanes[laneIndex]
policy, err = apply(StageExtract, lane.ID, lane.Extract, lane.ExtractExecutionClass)
if err != nil {
return err
}
lane.ExtractValidationPolicy = policy
policy, err = apply(StageMerge, lane.ID, lane.Merge, lane.MergeExecutionClass)
if err != nil {
return err
}
lane.MergeValidationPolicy = policy
policy, err = apply(StageNormalize, lane.ID, lane.Normalize, lane.NormalizeExecutionClass)
if err != nil {
return err
}
lane.NormalizeValidationPolicy = policy
}
}
return nil
}
func resolveBindings(bindings []ModuleBinding, defaultModule string, referenceSlotLabel string) ([]ModuleBinding, error) { func resolveBindings(bindings []ModuleBinding, defaultModule string, referenceSlotLabel string) ([]ModuleBinding, error) {
if len(bindings) == 0 { if len(bindings) == 0 {
return nil, nil return nil, nil
@@ -1549,27 +1631,31 @@ func selectedArtifactLanes(pipelineID string, artifacts map[string]ArtifactLaneP
func resolvedPipelineDigest(resolved ResolvedPipeline) (string, error) { func resolvedPipelineDigest(resolved ResolvedPipeline) (string, error) {
withoutDigest := struct { withoutDigest := struct {
ID string ID string
Input ModuleBinding ConfiguredValidationPolicy *ValidationPolicyOverride
InputExecutionClass contracts.ExecutionClass Input ModuleBinding
Chunk ModuleBinding InputExecutionClass contracts.ExecutionClass
ChunkExecutionClass contracts.ExecutionClass Chunk ModuleBinding
ChunkReferences ResolvedReferenceTarget ChunkExecutionClass contracts.ExecutionClass
Steps []ResolvedPipelineStep ChunkValidationPolicy ValidationPolicy
ValidatorChains []ResolvedValidatorChain ChunkReferences ResolvedReferenceTarget
Output ModuleBinding Steps []ResolvedPipelineStep
OutputExecutionClass contracts.ExecutionClass ValidatorChains []ResolvedValidatorChain
Output ModuleBinding
OutputExecutionClass contracts.ExecutionClass
}{ }{
ID: resolved.ID, ID: resolved.ID,
Input: resolved.Input, ConfiguredValidationPolicy: cloneValidationPolicyOverride(resolved.ConfiguredValidationPolicy),
InputExecutionClass: resolved.InputExecutionClass, ChunkValidationPolicy: resolved.ChunkValidationPolicy,
Chunk: resolved.Chunk, Input: resolved.Input,
ChunkExecutionClass: resolved.ChunkExecutionClass, InputExecutionClass: resolved.InputExecutionClass,
ChunkReferences: resolved.ChunkReferences, Chunk: resolved.Chunk,
Steps: resolved.Steps, ChunkExecutionClass: resolved.ChunkExecutionClass,
ValidatorChains: resolved.ValidatorChains, ChunkReferences: resolved.ChunkReferences,
Output: resolved.Output, Steps: resolved.Steps,
OutputExecutionClass: resolved.OutputExecutionClass, ValidatorChains: resolved.ValidatorChains,
Output: resolved.Output,
OutputExecutionClass: resolved.OutputExecutionClass,
} }
encoded, err := json.Marshal(withoutDigest) encoded, err := json.Marshal(withoutDigest)
if err != nil { if err != nil {

View File

@@ -0,0 +1,115 @@
package pipeline
import "fmt"
type ProducerStructuralFailureAction string
const (
ProducerStructuralFailureFailRun ProducerStructuralFailureAction = "fail_run"
ProducerStructuralFailureRejectOutput ProducerStructuralFailureAction = "reject_output"
)
type SemanticRejectionAction string
const (
SemanticRejectionFailRun SemanticRejectionAction = "fail_run"
SemanticRejectionRejectOutput SemanticRejectionAction = "reject_output"
)
type ValidatorFailureAction string
const (
ValidatorFailureWarnContinue ValidatorFailureAction = "warn_continue"
ValidatorFailureFailRun ValidatorFailureAction = "fail_run"
)
// ValidationPolicyOverride records only explicitly configured terminal-policy
// values. Nil fields inherit independently from the pipeline or application.
type ValidationPolicyOverride struct {
ProducerStructuralFailure *ProducerStructuralFailureAction `json:"producer_structural_failure,omitempty"`
SemanticRejection *SemanticRejectionAction `json:"semantic_rejection,omitempty"`
ValidatorFailure *ValidatorFailureAction `json:"validator_failure,omitempty"`
}
// ValidationPolicy is the concrete terminal policy retained by each resolved
// producer. It contains no inherited pointers.
type ValidationPolicy struct {
ProducerStructuralFailure ProducerStructuralFailureAction `json:"producer_structural_failure"`
SemanticRejection SemanticRejectionAction `json:"semantic_rejection"`
ValidatorFailure ValidatorFailureAction `json:"validator_failure"`
}
func DefaultValidationPolicy() ValidationPolicy {
return ValidationPolicy{
ProducerStructuralFailure: ProducerStructuralFailureFailRun,
SemanticRejection: SemanticRejectionFailRun,
ValidatorFailure: ValidatorFailureWarnContinue,
}
}
func (override ValidationPolicyOverride) Validate() error {
if override.ProducerStructuralFailure != nil {
switch *override.ProducerStructuralFailure {
case ProducerStructuralFailureFailRun, ProducerStructuralFailureRejectOutput:
default:
return fmt.Errorf("producer_structural_failure must be fail_run or reject_output")
}
}
if override.SemanticRejection != nil {
switch *override.SemanticRejection {
case SemanticRejectionFailRun, SemanticRejectionRejectOutput:
default:
return fmt.Errorf("semantic_rejection must be fail_run or reject_output")
}
}
if override.ValidatorFailure != nil {
switch *override.ValidatorFailure {
case ValidatorFailureWarnContinue, ValidatorFailureFailRun:
default:
return fmt.Errorf("validator_failure must be warn_continue or fail_run")
}
}
return nil
}
func ResolveValidationPolicy(binding, pipeline *ValidationPolicyOverride) ValidationPolicy {
resolved := DefaultValidationPolicy()
applyValidationPolicyOverride(&resolved, pipeline)
applyValidationPolicyOverride(&resolved, binding)
return resolved
}
func cloneValidationPolicyOverride(override *ValidationPolicyOverride) *ValidationPolicyOverride {
if override == nil {
return nil
}
out := *override
if override.ProducerStructuralFailure != nil {
value := *override.ProducerStructuralFailure
out.ProducerStructuralFailure = &value
}
if override.SemanticRejection != nil {
value := *override.SemanticRejection
out.SemanticRejection = &value
}
if override.ValidatorFailure != nil {
value := *override.ValidatorFailure
out.ValidatorFailure = &value
}
return &out
}
func applyValidationPolicyOverride(policy *ValidationPolicy, override *ValidationPolicyOverride) {
if override == nil {
return
}
if override.ProducerStructuralFailure != nil {
policy.ProducerStructuralFailure = *override.ProducerStructuralFailure
}
if override.SemanticRejection != nil {
policy.SemanticRejection = *override.SemanticRejection
}
if override.ValidatorFailure != nil {
policy.ValidatorFailure = *override.ValidatorFailure
}
}

View File

@@ -0,0 +1,103 @@
package pipeline
import (
"encoding/json"
"strings"
"testing"
)
func TestResolvePipelineAppliesValidationPolicyFieldByField(t *testing.T) {
profile := baselineProfile()
profile.ValidationPolicy = validationPolicyOverride(
ProducerStructuralFailureRejectOutput,
SemanticRejectionRejectOutput,
ValidatorFailureFailRun,
)
profile.Chunk.ValidationPolicy = validationPolicyOverride("", "", ValidatorFailureWarnContinue)
lane := profile.Artifacts["events"]
lane.Extract.ValidationPolicy = validationPolicyOverride("", SemanticRejectionFailRun, "")
lane.Merge.ValidationPolicy = validationPolicyOverride("", "", ValidatorFailureWarnContinue)
profile.Artifacts["events"] = lane
resolved, err := ResolvePipeline(profile, ResolveOptions{}, newProfileCatalog(t))
if err != nil {
t.Fatalf("ResolvePipeline() error = %v", err)
}
if got, want := resolved.ChunkValidationPolicy, (ValidationPolicy{ProducerStructuralFailure: ProducerStructuralFailureRejectOutput, SemanticRejection: SemanticRejectionRejectOutput, ValidatorFailure: ValidatorFailureWarnContinue}); got != want {
t.Fatalf("chunk validation policy = %#v, want %#v", got, want)
}
resolvedLane := resolved.Steps[0].ArtifactLanes[0]
if got, want := resolvedLane.ExtractValidationPolicy, (ValidationPolicy{ProducerStructuralFailure: ProducerStructuralFailureRejectOutput, SemanticRejection: SemanticRejectionFailRun, ValidatorFailure: ValidatorFailureFailRun}); got != want {
t.Fatalf("extract validation policy = %#v, want %#v", got, want)
}
if got, want := resolvedLane.MergeValidationPolicy, (ValidationPolicy{ProducerStructuralFailure: ProducerStructuralFailureRejectOutput, SemanticRejection: SemanticRejectionRejectOutput, ValidatorFailure: ValidatorFailureWarnContinue}); got != want {
t.Fatalf("merge validation policy = %#v, want %#v", got, want)
}
if got, want := resolvedLane.NormalizeValidationPolicy, (ValidationPolicy{ProducerStructuralFailure: ProducerStructuralFailureRejectOutput, SemanticRejection: SemanticRejectionRejectOutput, ValidatorFailure: ValidatorFailureFailRun}); got != want {
t.Fatalf("normalize validation policy = %#v, want %#v", got, want)
}
*profile.ValidationPolicy.SemanticRejection = SemanticRejectionFailRun
if got := resolved.ConfiguredValidationPolicy.SemanticRejection; got == nil || *got != SemanticRejectionRejectOutput {
t.Fatalf("resolved configured policy aliases profile: %#v", resolved.ConfiguredValidationPolicy)
}
}
func TestResolvePipelineRejectsStructuralPolicyOverrideOnDeterministicProducer(t *testing.T) {
profile := baselineProfile()
profile.Chunk.ValidationPolicy = validationPolicyOverride(ProducerStructuralFailureRejectOutput, "", "")
_, err := ResolvePipeline(profile, ResolveOptions{}, newProfileCatalog(t))
if err == nil || !strings.Contains(err.Error(), "producer_structural_failure") || !strings.Contains(err.Error(), "deterministic") {
t.Fatalf("ResolvePipeline() error = %v, want deterministic structural-policy rejection", err)
}
}
func TestValidationPolicyChangesResolvedDigestAndRoundTripsBindings(t *testing.T) {
left := baselineProfile()
right := baselineProfile()
right.ValidationPolicy = validationPolicyOverride("", SemanticRejectionRejectOutput, "")
leftResolved, err := ResolvePipeline(left, ResolveOptions{}, newProfileCatalog(t))
if err != nil {
t.Fatalf("ResolvePipeline(left) error = %v", err)
}
rightResolved, err := ResolvePipeline(right, ResolveOptions{}, newProfileCatalog(t))
if err != nil {
t.Fatalf("ResolvePipeline(right) error = %v", err)
}
if leftResolved.Digest == rightResolved.Digest {
t.Fatal("validation policy change did not change resolved digest")
}
binding := ModuleBinding{Module: "producer", ValidationPolicy: validationPolicyOverride(ProducerStructuralFailureRejectOutput, SemanticRejectionRejectOutput, ValidatorFailureFailRun)}
encoded, err := json.Marshal(binding)
if err != nil {
t.Fatalf("marshal binding: %v", err)
}
var decoded ModuleBinding
if err := json.Unmarshal(encoded, &decoded); err != nil {
t.Fatalf("unmarshal binding: %v", err)
}
if decoded.ValidationPolicy == nil || *decoded.ValidationPolicy.ProducerStructuralFailure != ProducerStructuralFailureRejectOutput || *decoded.ValidationPolicy.SemanticRejection != SemanticRejectionRejectOutput || *decoded.ValidationPolicy.ValidatorFailure != ValidatorFailureFailRun {
t.Fatalf("round-tripped validation policy = %#v", decoded.ValidationPolicy)
}
cloned := cloneModuleBinding(binding)
*binding.ValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
if got := *cloned.ValidationPolicy.ValidatorFailure; got != ValidatorFailureFailRun {
t.Fatalf("cloned validation policy aliases binding: %q", got)
}
}
func validationPolicyOverride(structural ProducerStructuralFailureAction, semantic SemanticRejectionAction, validator ValidatorFailureAction) *ValidationPolicyOverride {
policy := &ValidationPolicyOverride{}
if structural != "" {
policy.ProducerStructuralFailure = &structural
}
if semantic != "" {
policy.SemanticRejection = &semantic
}
if validator != "" {
policy.ValidatorFailure = &validator
}
return policy
}

View File

@@ -95,6 +95,7 @@ func cloneModuleBinding(binding ModuleBinding) ModuleBinding {
binding.Module = strings.TrimSpace(binding.Module) binding.Module = strings.TrimSpace(binding.Module)
binding.LLMProfile = strings.TrimSpace(binding.LLMProfile) binding.LLMProfile = strings.TrimSpace(binding.LLMProfile)
binding.StructuredOutputRepairAttempts = cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts) binding.StructuredOutputRepairAttempts = cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts)
binding.ValidationPolicy = cloneValidationPolicyOverride(binding.ValidationPolicy)
binding.Options = cloneOptions(binding.Options) binding.Options = cloneOptions(binding.Options)
if len(binding.References) > 0 { if len(binding.References) > 0 {
references := make(map[string]ReferenceSource, len(binding.References)) references := make(map[string]ReferenceSource, len(binding.References))