Add validation policy configuration

This commit is contained in:
2026-08-26 23:34:06 +00:00
parent 1c3da3e869
commit 9d0faabf61
14 changed files with 628 additions and 57 deletions

View File

@@ -223,6 +223,7 @@ pipelines:
| --- | --- | --- | --- |
| **llm_profile** | string | none | Optional non-empty default PromptKit profile ID for selected LLM-backed bindings and validators. An explicitly present blank value is invalid. |
| **structured_output_repair_attempts** | integer | prompt-owned (1 in maintained production prompts) | Optional structural-repair limit from 0 through 3 for selected LLM-backed bindings and validators. Omission leaves the prompt's declared policy in control; explicit 0 disables structural repair at that scope. |
| **validation_policy** | object | see below | Optional terminal policy defaults for producer validation. Its fields inherit independently into chunk, extract, merge, and normalize bindings. |
| **input** | module binding | none | Required. |
| **chunk** | module binding | **generic** | Optional. |
| **output** | module binding | **json** | Optional. |
@@ -251,6 +252,24 @@ explicit `null` and non-integer values are invalid. An explicit value on a
deterministic binding or validator is invalid, while a pipeline value simply
does not apply to deterministic selections.
`validation_policy` records terminal-policy metadata for a complete producer
attempt and validation chain. It may appear on a pipeline or a **chunk**,
**extract**, **merge**, or **normalize** module binding; input, output, and
validator bindings reject it. Every field is optional and resolves in binding,
pipeline, then application-default order. The current runner records the
resolved values but does not yet consume them for terminal disposition:
| Field | Values | Default |
| --- | --- | --- |
| **producer_structural_failure** | **fail_run**, **reject_output** | **fail_run** |
| **semantic_rejection** | **fail_run**, **reject_output** | **fail_run** |
| **validator_failure** | **warn_continue**, **fail_run** | **warn_continue** |
The policy object and its fields must be non-null, and unknown fields are
rejected. A deterministic producer may not explicitly set
**producer_structural_failure** on its binding, although a pipeline-level
default remains valid for pipelines that include LLM-backed producers.
A lane has these fields:
| Field | Type | Default | Rules |
@@ -289,6 +308,7 @@ extract:
| **module** | string | none | Required for an object binding. Must be a registered compatible key. |
| **llm_profile** | string | none | Optional non-empty PromptKit profile ID for an LLM-backed binding. It overrides the pipeline default unless the run supplies **--llm-profile**. |
| **structured_output_repair_attempts** | integer | pipeline or prompt-owned (1 in maintained production prompts) | Optional structural-repair limit from 0 through 3 for an LLM-backed binding. It overrides the pipeline value; explicit 0 disables structural repair. |
| **validation_policy** | object | pipeline or application defaults | Optional field-by-field terminal-policy override for a chunk, extract, merge, or normalize binding. |
| **retries** | integer | 0 | Non-negative additional attempts for chunk, extract, merge, and normalize bindings. |
| **options** | object | none | Must satisfy the selected module. |
| **references** | map | none | Valid only on chunk, extract, merge, and normalize bindings. |
@@ -298,6 +318,7 @@ Omitting **validators** uses the registered chain. **validators: []** selects
an empty chain; a non-empty list replaces the chain in the listed order.
Validator bindings accept only **module**, **llm_profile**,
**structured_output_repair_attempts**, and **options**. They reject
**validation_policy**,
**references**, **retries**, and nested **validators**. Deterministic validators
reject explicit **llm_profile** and **structured_output_repair_attempts**.
Deterministic module bindings also reject those explicit fields.

View File

@@ -74,6 +74,12 @@ profile-free, and no second inheritance decision occurs during execution. The
public field definitions and precedence are owned by
[Configuration](../config.md#pipelines).
The resolver retains configured `validation_policy` overrides and derives one
detached concrete terminal policy for the chunk producer and every lane's
extract, merge, and normalize producers. That field-by-field inheritance is
complete before preparation, and the effective values contribute to pipeline
and checkpoint identity; execution does not interpret configuration defaults.
The framework resolver supplies defaults, selects lanes, resolves validator
chains, checks registered module and artifact compatibility, validates module
options, and returns the fixed ordered pipeline shape. The resulting

View File

@@ -189,7 +189,7 @@ The generic contracts can safely carry correction input and exact candidate
material across every producer stage, no PromptKit type crosses the boundary,
and existing runtime behavior is unchanged. This stage is one Terra prompt.
## Stage 3 — Add Validation Policy Configuration And Resolution
## Stage 3 — Add Validation Policy Configuration And Resolution
### Goal

View File

@@ -116,6 +116,7 @@ func (c *ConcurrencyConfig) recomputeStageWorkerDefaults() {
func clonePipelineProfile(in pipeline.PipelineProfile) pipeline.PipelineProfile {
out := in
out.ValidationPolicy = cloneValidationPolicyOverride(in.ValidationPolicy)
if in.StructuredOutputRepairAttempts != nil {
value := *in.StructuredOutputRepairAttempts
out.StructuredOutputRepairAttempts = &value
@@ -200,6 +201,7 @@ func cloneReferenceSource(in pipeline.ReferenceSource) pipeline.ReferenceSource
func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
out := in
out.ValidationPolicy = cloneValidationPolicyOverride(in.ValidationPolicy)
if in.StructuredOutputRepairAttempts != nil {
value := *in.StructuredOutputRepairAttempts
out.StructuredOutputRepairAttempts = &value
@@ -212,6 +214,26 @@ func cloneModuleBinding(in pipeline.ModuleBinding) pipeline.ModuleBinding {
return out
}
func cloneValidationPolicyOverride(in *pipeline.ValidationPolicyOverride) *pipeline.ValidationPolicyOverride {
if in == nil {
return nil
}
out := *in
if in.ProducerStructuralFailure != nil {
value := *in.ProducerStructuralFailure
out.ProducerStructuralFailure = &value
}
if in.SemanticRejection != nil {
value := *in.SemanticRejection
out.SemanticRejection = &value
}
if in.ValidatorFailure != nil {
value := *in.ValidatorFailure
out.ValidatorFailure = &value
}
return &out
}
func cloneValidatorOverride(in pipeline.ValidatorOverride) pipeline.ValidatorOverride {
out := pipeline.ValidatorOverride{Set: in.Set}
if len(in.Validators) > 0 {

View File

@@ -37,6 +37,7 @@ type FilePromptKitLocalBackendConfig struct {
type FilePipelineProfile struct {
LLMProfile *string `yaml:"llm_profile,omitempty"`
StructuredOutputRepairAttempts *int `yaml:"structured_output_repair_attempts,omitempty"`
ValidationPolicy *pipeline.ValidationPolicyOverride `yaml:"validation_policy,omitempty"`
Input fileModuleBinding `yaml:"input"`
Chunk *fileModuleBinding `yaml:"chunk,omitempty"`
Artifacts map[string]FileArtifactLaneProfile `yaml:"artifacts,omitempty"`
@@ -55,12 +56,19 @@ func (p *FilePipelineProfile) UnmarshalYAML(node *yaml.Node) error {
type plainFilePipelineProfile FilePipelineProfile
var decoded plainFilePipelineProfile
seen, err := decodeKnownMapping(node, &decoded, map[string]struct{}{
"llm_profile": {}, "structured_output_repair_attempts": {}, "input": {}, "chunk": {}, "artifacts": {}, "steps": {}, "output": {}, "references": {},
"llm_profile": {}, "structured_output_repair_attempts": {}, "validation_policy": {}, "input": {}, "chunk": {}, "artifacts": {}, "steps": {}, "output": {}, "references": {},
}, "pipeline profile")
if err != nil {
return err
}
*p = FilePipelineProfile(decoded)
if validationPolicyNode, ok := mappingValue(node, "validation_policy"); ok {
policy, err := parseValidationPolicy(validationPolicyNode, "pipeline profile")
if err != nil {
return err
}
p.ValidationPolicy = policy
}
_, p.artifactsSet = seen["artifacts"]
_, p.stepsSet = seen["steps"]
_, p.llmProfileSet = seen["llm_profile"]
@@ -154,6 +162,7 @@ type fileModuleBinding struct {
Module string
LLMProfile string
StructuredOutputRepairAttempts *int
ValidationPolicy *pipeline.ValidationPolicyOverride
Retries int
Options map[string]any
References map[string]fileReferenceSource
@@ -250,9 +259,14 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
b.Module = strings.TrimSpace(module)
return nil
case yaml.MappingNode:
seen := make(map[string]struct{}, len(node.Content)/2)
for i := 0; i < len(node.Content); i += 2 {
keyNode := node.Content[i]
valueNode := node.Content[i+1]
if _, exists := seen[keyNode.Value]; exists {
return fmt.Errorf("module binding field %q is duplicated", keyNode.Value)
}
seen[keyNode.Value] = struct{}{}
switch keyNode.Value {
case "module":
var module string
@@ -275,6 +289,12 @@ func (b *fileModuleBinding) UnmarshalYAML(node *yaml.Node) error {
return err
}
b.StructuredOutputRepairAttempts = attempts
case "validation_policy":
policy, err := parseValidationPolicy(valueNode, "module binding")
if err != nil {
return err
}
b.ValidationPolicy = policy
case "retries":
var retries int
if err := valueNode.Decode(&retries); err != nil {
@@ -318,6 +338,7 @@ func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
Module: strings.TrimSpace(b.Module),
LLMProfile: strings.TrimSpace(b.LLMProfile),
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(b.StructuredOutputRepairAttempts),
ValidationPolicy: cloneValidationPolicyOverride(b.ValidationPolicy),
Retries: b.Retries,
Options: cloneOptions(b.Options),
References: fileReferenceSourcesToPipeline(b.References),
@@ -325,6 +346,51 @@ func (b fileModuleBinding) toPipelineBinding() pipeline.ModuleBinding {
}
}
func mappingValue(node *yaml.Node, key string) (*yaml.Node, bool) {
for i := 0; i < len(node.Content); i += 2 {
if node.Content[i].Value == key {
return node.Content[i+1], true
}
}
return nil, false
}
func parseValidationPolicy(node *yaml.Node, context string) (*pipeline.ValidationPolicyOverride, error) {
if node == nil || node.Tag == "!!null" || node.Kind != yaml.MappingNode {
return nil, fmt.Errorf("%s validation_policy must be an object", context)
}
policy := &pipeline.ValidationPolicyOverride{}
seen := make(map[string]struct{}, len(node.Content)/2)
for i := 0; i < len(node.Content); i += 2 {
key := node.Content[i].Value
value := node.Content[i+1]
if _, exists := seen[key]; exists {
return nil, fmt.Errorf("%s validation_policy field %q is duplicated", context, key)
}
seen[key] = struct{}{}
if value.Tag == "!!null" || value.Kind != yaml.ScalarNode || value.Tag != "!!str" {
return nil, fmt.Errorf("%s validation_policy.%s must be a string", context, key)
}
switch key {
case "producer_structural_failure":
value := pipeline.ProducerStructuralFailureAction(value.Value)
policy.ProducerStructuralFailure = &value
case "semantic_rejection":
value := pipeline.SemanticRejectionAction(value.Value)
policy.SemanticRejection = &value
case "validator_failure":
value := pipeline.ValidatorFailureAction(value.Value)
policy.ValidatorFailure = &value
default:
return nil, fmt.Errorf("field %s not found in %s validation_policy", key, context)
}
}
if err := policy.Validate(); err != nil {
return nil, fmt.Errorf("%s validation_policy: %w", context, err)
}
return policy, nil
}
func validateStructuredOutputRepairAttemptsNode(node *yaml.Node, context string) error {
if node.Kind != yaml.MappingNode {
return fmt.Errorf("%s must be an object", context)
@@ -573,6 +639,7 @@ func (c *Config) applyFileConfigWithLookup(fileCfg FileConfig, lookup func(strin
ID: pipelineID,
LLMProfile: llmProfile,
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(filePipeline.StructuredOutputRepairAttempts),
ValidationPolicy: cloneValidationPolicyOverride(filePipeline.ValidationPolicy),
Input: filePipeline.Input.toPipelineBinding(),
Artifacts: make(map[string]pipeline.ArtifactLaneProfile, len(filePipeline.Artifacts)),
References: fileReferenceSourcesToPipeline(filePipeline.References),

View File

@@ -32,6 +32,7 @@ func (e EffectiveConfig) RedactedResolvedPipelinePayload() pipeline.ResolvedPipe
func cloneResolvedPipeline(in pipeline.ResolvedPipeline) pipeline.ResolvedPipeline {
out := in
out.ConfiguredValidationPolicy = cloneValidationPolicyOverride(in.ConfiguredValidationPolicy)
out.Input = redactBinding(cloneModuleBinding(in.Input))
out.Chunk = redactBinding(cloneModuleBinding(in.Chunk))
out.ChunkReferences = pipeline.CloneReferenceTarget(in.ChunkReferences)

View File

@@ -225,6 +225,39 @@ func TestRedactedResolvedPipelinePayloadHandlesTypedOptionContainers(t *testing.
}
}
func TestRedactedEffectiveConfigPayloadOwnsValidationPolicies(t *testing.T) {
semantic := pipeline.SemanticRejectionRejectOutput
validator := pipeline.ValidatorFailureFailRun
configured := &pipeline.ValidationPolicyOverride{SemanticRejection: &semantic, ValidatorFailure: &validator}
effective := EffectiveConfig{
Config: Config{Pipelines: map[string]pipeline.PipelineProfile{
"main": {ValidationPolicy: configured},
}},
ResolvedPipeline: pipeline.ResolvedPipeline{
ConfiguredValidationPolicy: configured,
ChunkValidationPolicy: pipeline.ValidationPolicy{
ProducerStructuralFailure: pipeline.ProducerStructuralFailureFailRun,
SemanticRejection: pipeline.SemanticRejectionRejectOutput,
ValidatorFailure: pipeline.ValidatorFailureFailRun,
},
},
}
payload := effective.RedactedSummaryPayload().(EffectiveConfig)
encoded, err := json.Marshal(payload)
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(encoded), `"configured_validation_policy":{"semantic_rejection":"reject_output","validator_failure":"fail_run"}`) || !strings.Contains(string(encoded), `"chunk_validation_policy":{"producer_structural_failure":"fail_run","semantic_rejection":"reject_output","validator_failure":"fail_run"}`) {
t.Fatalf("redacted payload omitted validation policy: %s", encoded)
}
*payload.Config.Pipelines["main"].ValidationPolicy.SemanticRejection = pipeline.SemanticRejectionFailRun
*payload.ResolvedPipeline.ConfiguredValidationPolicy.ValidatorFailure = pipeline.ValidatorFailureWarnContinue
if *effective.Config.Pipelines["main"].ValidationPolicy.SemanticRejection != pipeline.SemanticRejectionRejectOutput || *effective.ResolvedPipeline.ConfiguredValidationPolicy.ValidatorFailure != pipeline.ValidatorFailureFailRun {
t.Fatal("redacted payload aliases validation policy")
}
}
func redactionTestBinding(name string) pipeline.ModuleBinding {
return pipeline.ModuleBinding{
Module: "safe-" + name,

View File

@@ -119,6 +119,11 @@ func validatePipelineProfiles(profiles map[string]pipeline.PipelineProfile) erro
if err := validateStructuredOutputRepairAttempts(fmt.Sprintf("pipeline %q", id), profile.StructuredOutputRepairAttempts); err != nil {
return err
}
if profile.ValidationPolicy != nil {
if err := profile.ValidationPolicy.Validate(); err != nil {
return fmt.Errorf("pipeline %q validation_policy: %w", id, err)
}
}
if err := validateBinding(id, "", "input", profile.Input, false); err != nil {
return err
}
@@ -198,6 +203,16 @@ func validateBinding(
binding pipeline.ModuleBinding,
referencesAllowed bool,
) error {
if binding.ValidationPolicy != nil {
switch slot {
case "chunk", "extract", "merge", "normalize":
default:
return fmt.Errorf("%s validation_policy is not supported", referenceContext(pipelineID, laneID, slot))
}
if err := binding.ValidationPolicy.Validate(); err != nil {
return fmt.Errorf("%s validation_policy: %w", referenceContext(pipelineID, laneID, slot), err)
}
}
if err := validateStructuredOutputRepairAttempts(referenceContext(pipelineID, laneID, slot), binding.StructuredOutputRepairAttempts); err != nil {
return err
}
@@ -255,6 +270,9 @@ func validateValidatorOverride(pipelineID string, laneID string, slot string, ov
if validator.Validators.Set {
return fmt.Errorf("%s nested validators are not supported", context)
}
if validator.ValidationPolicy != nil {
return fmt.Errorf("%s validation_policy is not supported", context)
}
if validator.Retries != 0 {
return fmt.Errorf("%s retries are not supported", context)
}

View File

@@ -0,0 +1,97 @@
package config
import (
"strings"
"testing"
"gitea.maximumdirect.net/eric/notarius/internal/framework/pipeline"
)
func TestValidationPolicyFileConfigurationIsStrictAndPresenceAware(t *testing.T) {
const valid = `version: 4
pipelines:
main:
validation_policy:
producer_structural_failure: reject_output
semantic_rejection: fail_run
input: seriatim
chunk:
module: generic
validation_policy:
validator_failure: fail_run
artifacts:
lane:
extract:
module: extract
validation_policy:
semantic_rejection: reject_output
`
cfg := applyFileConfig(t, valid)
profile := cfg.Pipelines["main"]
if profile.ValidationPolicy == nil || profile.ValidationPolicy.ProducerStructuralFailure == nil || *profile.ValidationPolicy.ProducerStructuralFailure != pipeline.ProducerStructuralFailureRejectOutput || profile.ValidationPolicy.SemanticRejection == nil || *profile.ValidationPolicy.SemanticRejection != pipeline.SemanticRejectionFailRun || profile.ValidationPolicy.ValidatorFailure != nil {
t.Fatalf("pipeline validation policy = %#v", profile.ValidationPolicy)
}
if profile.Chunk.ValidationPolicy == nil || profile.Chunk.ValidationPolicy.ValidatorFailure == nil || *profile.Chunk.ValidationPolicy.ValidatorFailure != pipeline.ValidatorFailureFailRun {
t.Fatalf("chunk validation policy = %#v", profile.Chunk.ValidationPolicy)
}
lane := profile.Artifacts["lane"]
if lane.Extract.ValidationPolicy == nil || lane.Extract.ValidationPolicy.SemanticRejection == nil || *lane.Extract.ValidationPolicy.SemanticRejection != pipeline.SemanticRejectionRejectOutput {
t.Fatalf("extract validation policy = %#v", lane.Extract.ValidationPolicy)
}
for _, test := range []struct {
name string
yaml string
}{
{"null object", strings.Replace(valid, "validation_policy:\n producer_structural_failure: reject_output\n semantic_rejection: fail_run", "validation_policy: null", 1)},
{"null field", strings.Replace(valid, "semantic_rejection: fail_run", "semantic_rejection: null", 1)},
{"unknown field", strings.Replace(valid, "semantic_rejection: fail_run", "unknown: fail_run", 1)},
{"duplicate field", strings.Replace(valid, "semantic_rejection: fail_run", "semantic_rejection: fail_run\n semantic_rejection: reject_output", 1)},
{"invalid enum", strings.Replace(valid, "semantic_rejection: fail_run", "semantic_rejection: continue", 1)},
} {
t.Run(test.name, func(t *testing.T) {
if _, err := ParseFileConfigYAML([]byte(test.yaml)); err == nil {
t.Fatal("ParseFileConfigYAML() error = nil, want strict validation-policy rejection")
}
})
}
}
func TestValidationPolicyPlacementRules(t *testing.T) {
policy := &pipeline.ValidationPolicyOverride{}
semantic := pipeline.SemanticRejectionRejectOutput
policy.SemanticRejection = &semantic
base := pipeline.PipelineProfile{
ID: "main",
Input: pipeline.Binding("input"),
Artifacts: map[string]pipeline.ArtifactLaneProfile{
"lane": {Extract: pipeline.Binding("extract")},
},
}
for _, test := range []struct {
name string
mutate func(*pipeline.PipelineProfile)
}{
{"input", func(profile *pipeline.PipelineProfile) { profile.Input.ValidationPolicy = policy }},
{"output", func(profile *pipeline.PipelineProfile) {
profile.Output = pipeline.Binding("output")
profile.Output.ValidationPolicy = policy
}},
{"validator", func(profile *pipeline.PipelineProfile) {
lane := profile.Artifacts["lane"]
lane.Extract.Validators = pipeline.ValidatorOverride{Set: true, Validators: []pipeline.ModuleBinding{{Module: "validator", ValidationPolicy: policy}}}
profile.Artifacts["lane"] = lane
}},
} {
t.Run(test.name, func(t *testing.T) {
profile := base
profile.Artifacts = map[string]pipeline.ArtifactLaneProfile{"lane": base.Artifacts["lane"]}
test.mutate(&profile)
cfg := Default()
cfg.Pipelines = map[string]pipeline.PipelineProfile{"main": profile}
if err := cfg.Validate(); err == nil || !strings.Contains(err.Error(), "validation_policy") {
t.Fatalf("Config.Validate() error = %v, want placement rejection", err)
}
})
}
}

View File

@@ -406,6 +406,7 @@ func validateRegistrySet(resolved ResolvedPipeline, registries Registries) error
func cloneResolvedPipeline(in ResolvedPipeline) ResolvedPipeline {
out := in
out.ConfiguredValidationPolicy = cloneValidationPolicyOverride(in.ConfiguredValidationPolicy)
out.Input = cloneModuleBinding(in.Input)
out.Chunk = cloneModuleBinding(in.Chunk)
out.Output = cloneModuleBinding(in.Output)

View File

@@ -24,6 +24,7 @@ type ModuleBinding struct {
Module string `json:"module"`
LLMProfile string `json:"llm_profile,omitempty"`
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
ValidationPolicy *ValidationPolicyOverride `json:"validation_policy,omitempty"`
Retries int `json:"retries,omitempty"`
Options map[string]any `json:"options,omitempty"`
References map[string]ReferenceSource `json:"references,omitempty"`
@@ -84,6 +85,7 @@ func (binding ModuleBinding) MarshalJSON() ([]byte, error) {
Module string `json:"module"`
LLMProfile string `json:"llm_profile,omitempty"`
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
ValidationPolicy *ValidationPolicyOverride `json:"validation_policy,omitempty"`
Retries int `json:"retries,omitempty"`
Options map[string]any `json:"options,omitempty"`
References map[string]ReferenceSource `json:"references,omitempty"`
@@ -93,6 +95,7 @@ func (binding ModuleBinding) MarshalJSON() ([]byte, error) {
Module: binding.Module,
LLMProfile: binding.LLMProfile,
StructuredOutputRepairAttempts: binding.StructuredOutputRepairAttempts,
ValidationPolicy: cloneValidationPolicyOverride(binding.ValidationPolicy),
Retries: binding.Retries,
Options: binding.Options,
References: binding.References,
@@ -112,6 +115,7 @@ func (binding *ModuleBinding) UnmarshalJSON(data []byte) error {
Module string `json:"module"`
LLMProfile string `json:"llm_profile,omitempty"`
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
ValidationPolicy *ValidationPolicyOverride `json:"validation_policy,omitempty"`
Retries int `json:"retries,omitempty"`
Options map[string]any `json:"options,omitempty"`
References map[string]ReferenceSource `json:"references,omitempty"`
@@ -125,6 +129,7 @@ func (binding *ModuleBinding) UnmarshalJSON(data []byte) error {
Module: decoded.Module,
LLMProfile: decoded.LLMProfile,
StructuredOutputRepairAttempts: decoded.StructuredOutputRepairAttempts,
ValidationPolicy: cloneValidationPolicyOverride(decoded.ValidationPolicy),
Retries: decoded.Retries,
Options: decoded.Options,
References: decoded.References,
@@ -158,6 +163,7 @@ type PipelineProfile struct {
ID string `json:"id"`
LLMProfile string `json:"llm_profile,omitempty"`
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
ValidationPolicy *ValidationPolicyOverride `json:"validation_policy,omitempty"`
Input ModuleBinding `json:"input"`
Chunk ModuleBinding `json:"chunk,omitempty"`
Artifacts map[string]ArtifactLaneProfile `json:"artifacts"`
@@ -207,10 +213,13 @@ type ResolvedArtifactLane struct {
ArtifactSchemaDigest string `json:"artifact_schema_digest,omitempty"`
Extract ModuleBinding
ExtractExecutionClass contracts.ExecutionClass `json:"extract_execution_class"`
ExtractValidationPolicy ValidationPolicy `json:"extract_validation_policy"`
Merge ModuleBinding
MergeExecutionClass contracts.ExecutionClass `json:"merge_execution_class"`
MergeValidationPolicy ValidationPolicy `json:"merge_validation_policy"`
Normalize ModuleBinding
NormalizeExecutionClass contracts.ExecutionClass `json:"normalize_execution_class"`
NormalizeValidationPolicy ValidationPolicy `json:"normalize_validation_policy"`
Validators []ModuleBinding
ExtractReferences ResolvedReferenceTarget `json:"extract_references"`
MergeReferences ResolvedReferenceTarget `json:"merge_references"`
@@ -239,10 +248,12 @@ type ResolvedValidator struct {
type ResolvedPipeline struct {
ID string
Digest string
ConfiguredValidationPolicy *ValidationPolicyOverride `json:"configured_validation_policy,omitempty"`
Input ModuleBinding
InputExecutionClass contracts.ExecutionClass `json:"input_execution_class"`
Chunk ModuleBinding
ChunkExecutionClass contracts.ExecutionClass `json:"chunk_execution_class"`
ChunkValidationPolicy ValidationPolicy `json:"chunk_validation_policy"`
ChunkReferences ResolvedReferenceTarget `json:"chunk_references"`
Steps []ResolvedPipelineStep
ValidatorChains []ResolvedValidatorChain `json:"validator_chains"`
@@ -415,6 +426,7 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
}
resolved := ResolvedPipeline{
ID: pipelineID,
ConfiguredValidationPolicy: cloneValidationPolicyOverride(profile.ValidationPolicy),
Input: input,
InputExecutionClass: inputModuleSpec.ExecutionClass,
Chunk: chunk,
@@ -493,6 +505,9 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
if err := applyEffectiveStructuredOutputRepairAttempts(&resolved, profile.StructuredOutputRepairAttempts); err != nil {
return ResolvedPipeline{}, err
}
if err := applyEffectiveValidationPolicies(&resolved, profile.ValidationPolicy); err != nil {
return ResolvedPipeline{}, err
}
if err := validateResolvedOptions(resolved, catalog, configuredLaneIDs); err != nil {
return ResolvedPipeline{}, err
}
@@ -1328,6 +1343,7 @@ func resolveBinding(binding ModuleBinding, defaultModule string, referenceSlotLa
Module: module,
LLMProfile: llmProfile,
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts),
ValidationPolicy: cloneValidationPolicyOverride(binding.ValidationPolicy),
Retries: binding.Retries,
Options: cloneOptions(binding.Options),
References: references,
@@ -1456,6 +1472,72 @@ func applyEffectiveStructuredOutputRepairAttempts(resolved *ResolvedPipeline, pi
return nil
}
func applyEffectiveValidationPolicies(resolved *ResolvedPipeline, pipelinePolicy *ValidationPolicyOverride) error {
apply := func(stage ModuleStage, laneID string, binding ModuleBinding, executionClass contracts.ExecutionClass) (ValidationPolicy, error) {
binding.ValidationPolicy = cloneValidationPolicyOverride(binding.ValidationPolicy)
if binding.ValidationPolicy != nil {
if err := binding.ValidationPolicy.Validate(); err != nil {
return ValidationPolicy{}, fmt.Errorf("pipeline %q %s validation_policy: %w", resolved.ID, stage, err)
}
if executionClass == contracts.ExecutionClassDeterministic && binding.ValidationPolicy.ProducerStructuralFailure != nil {
if laneID == "" {
return ValidationPolicy{}, fmt.Errorf("pipeline %q %s %q assigns producer_structural_failure to deterministic module", resolved.ID, stage, binding.Module)
}
return ValidationPolicy{}, fmt.Errorf("pipeline %q lane %q %s %q assigns producer_structural_failure to deterministic module", resolved.ID, laneID, stage, binding.Module)
}
}
return ResolveValidationPolicy(binding.ValidationPolicy, pipelinePolicy), nil
}
if pipelinePolicy != nil {
if err := pipelinePolicy.Validate(); err != nil {
return fmt.Errorf("pipeline %q validation_policy: %w", resolved.ID, err)
}
}
if resolved.Input.ValidationPolicy != nil {
return fmt.Errorf("pipeline %q input validation_policy is not supported", resolved.ID)
}
if resolved.Output.ValidationPolicy != nil {
return fmt.Errorf("pipeline %q output validation_policy is not supported", resolved.ID)
}
for _, chain := range resolved.ValidatorChains {
for _, validator := range chain.Validators {
if validator.Binding.ValidationPolicy != nil {
if chain.LaneID == "" {
return fmt.Errorf("pipeline %q %s validator %q validation_policy is not supported", resolved.ID, chain.Stage, validator.Binding.Module)
}
return fmt.Errorf("pipeline %q lane %q %s validator %q validation_policy is not supported", resolved.ID, chain.LaneID, chain.Stage, validator.Binding.Module)
}
}
}
policy, err := apply(StageChunk, "", resolved.Chunk, resolved.ChunkExecutionClass)
if err != nil {
return err
}
resolved.ChunkValidationPolicy = policy
for stepIndex := range resolved.Steps {
for laneIndex := range resolved.Steps[stepIndex].ArtifactLanes {
lane := &resolved.Steps[stepIndex].ArtifactLanes[laneIndex]
policy, err = apply(StageExtract, lane.ID, lane.Extract, lane.ExtractExecutionClass)
if err != nil {
return err
}
lane.ExtractValidationPolicy = policy
policy, err = apply(StageMerge, lane.ID, lane.Merge, lane.MergeExecutionClass)
if err != nil {
return err
}
lane.MergeValidationPolicy = policy
policy, err = apply(StageNormalize, lane.ID, lane.Normalize, lane.NormalizeExecutionClass)
if err != nil {
return err
}
lane.NormalizeValidationPolicy = policy
}
}
return nil
}
func resolveBindings(bindings []ModuleBinding, defaultModule string, referenceSlotLabel string) ([]ModuleBinding, error) {
if len(bindings) == 0 {
return nil, nil
@@ -1550,10 +1632,12 @@ func selectedArtifactLanes(pipelineID string, artifacts map[string]ArtifactLaneP
func resolvedPipelineDigest(resolved ResolvedPipeline) (string, error) {
withoutDigest := struct {
ID string
ConfiguredValidationPolicy *ValidationPolicyOverride
Input ModuleBinding
InputExecutionClass contracts.ExecutionClass
Chunk ModuleBinding
ChunkExecutionClass contracts.ExecutionClass
ChunkValidationPolicy ValidationPolicy
ChunkReferences ResolvedReferenceTarget
Steps []ResolvedPipelineStep
ValidatorChains []ResolvedValidatorChain
@@ -1561,6 +1645,8 @@ func resolvedPipelineDigest(resolved ResolvedPipeline) (string, error) {
OutputExecutionClass contracts.ExecutionClass
}{
ID: resolved.ID,
ConfiguredValidationPolicy: cloneValidationPolicyOverride(resolved.ConfiguredValidationPolicy),
ChunkValidationPolicy: resolved.ChunkValidationPolicy,
Input: resolved.Input,
InputExecutionClass: resolved.InputExecutionClass,
Chunk: resolved.Chunk,

View File

@@ -0,0 +1,115 @@
package pipeline
import "fmt"
type ProducerStructuralFailureAction string
const (
ProducerStructuralFailureFailRun ProducerStructuralFailureAction = "fail_run"
ProducerStructuralFailureRejectOutput ProducerStructuralFailureAction = "reject_output"
)
type SemanticRejectionAction string
const (
SemanticRejectionFailRun SemanticRejectionAction = "fail_run"
SemanticRejectionRejectOutput SemanticRejectionAction = "reject_output"
)
type ValidatorFailureAction string
const (
ValidatorFailureWarnContinue ValidatorFailureAction = "warn_continue"
ValidatorFailureFailRun ValidatorFailureAction = "fail_run"
)
// ValidationPolicyOverride records only explicitly configured terminal-policy
// values. Nil fields inherit independently from the pipeline or application.
type ValidationPolicyOverride struct {
ProducerStructuralFailure *ProducerStructuralFailureAction `json:"producer_structural_failure,omitempty"`
SemanticRejection *SemanticRejectionAction `json:"semantic_rejection,omitempty"`
ValidatorFailure *ValidatorFailureAction `json:"validator_failure,omitempty"`
}
// ValidationPolicy is the concrete terminal policy retained by each resolved
// producer. It contains no inherited pointers.
type ValidationPolicy struct {
ProducerStructuralFailure ProducerStructuralFailureAction `json:"producer_structural_failure"`
SemanticRejection SemanticRejectionAction `json:"semantic_rejection"`
ValidatorFailure ValidatorFailureAction `json:"validator_failure"`
}
func DefaultValidationPolicy() ValidationPolicy {
return ValidationPolicy{
ProducerStructuralFailure: ProducerStructuralFailureFailRun,
SemanticRejection: SemanticRejectionFailRun,
ValidatorFailure: ValidatorFailureWarnContinue,
}
}
func (override ValidationPolicyOverride) Validate() error {
if override.ProducerStructuralFailure != nil {
switch *override.ProducerStructuralFailure {
case ProducerStructuralFailureFailRun, ProducerStructuralFailureRejectOutput:
default:
return fmt.Errorf("producer_structural_failure must be fail_run or reject_output")
}
}
if override.SemanticRejection != nil {
switch *override.SemanticRejection {
case SemanticRejectionFailRun, SemanticRejectionRejectOutput:
default:
return fmt.Errorf("semantic_rejection must be fail_run or reject_output")
}
}
if override.ValidatorFailure != nil {
switch *override.ValidatorFailure {
case ValidatorFailureWarnContinue, ValidatorFailureFailRun:
default:
return fmt.Errorf("validator_failure must be warn_continue or fail_run")
}
}
return nil
}
func ResolveValidationPolicy(binding, pipeline *ValidationPolicyOverride) ValidationPolicy {
resolved := DefaultValidationPolicy()
applyValidationPolicyOverride(&resolved, pipeline)
applyValidationPolicyOverride(&resolved, binding)
return resolved
}
func cloneValidationPolicyOverride(override *ValidationPolicyOverride) *ValidationPolicyOverride {
if override == nil {
return nil
}
out := *override
if override.ProducerStructuralFailure != nil {
value := *override.ProducerStructuralFailure
out.ProducerStructuralFailure = &value
}
if override.SemanticRejection != nil {
value := *override.SemanticRejection
out.SemanticRejection = &value
}
if override.ValidatorFailure != nil {
value := *override.ValidatorFailure
out.ValidatorFailure = &value
}
return &out
}
func applyValidationPolicyOverride(policy *ValidationPolicy, override *ValidationPolicyOverride) {
if override == nil {
return
}
if override.ProducerStructuralFailure != nil {
policy.ProducerStructuralFailure = *override.ProducerStructuralFailure
}
if override.SemanticRejection != nil {
policy.SemanticRejection = *override.SemanticRejection
}
if override.ValidatorFailure != nil {
policy.ValidatorFailure = *override.ValidatorFailure
}
}

View File

@@ -0,0 +1,103 @@
package pipeline
import (
"encoding/json"
"strings"
"testing"
)
func TestResolvePipelineAppliesValidationPolicyFieldByField(t *testing.T) {
profile := baselineProfile()
profile.ValidationPolicy = validationPolicyOverride(
ProducerStructuralFailureRejectOutput,
SemanticRejectionRejectOutput,
ValidatorFailureFailRun,
)
profile.Chunk.ValidationPolicy = validationPolicyOverride("", "", ValidatorFailureWarnContinue)
lane := profile.Artifacts["events"]
lane.Extract.ValidationPolicy = validationPolicyOverride("", SemanticRejectionFailRun, "")
lane.Merge.ValidationPolicy = validationPolicyOverride("", "", ValidatorFailureWarnContinue)
profile.Artifacts["events"] = lane
resolved, err := ResolvePipeline(profile, ResolveOptions{}, newProfileCatalog(t))
if err != nil {
t.Fatalf("ResolvePipeline() error = %v", err)
}
if got, want := resolved.ChunkValidationPolicy, (ValidationPolicy{ProducerStructuralFailure: ProducerStructuralFailureRejectOutput, SemanticRejection: SemanticRejectionRejectOutput, ValidatorFailure: ValidatorFailureWarnContinue}); got != want {
t.Fatalf("chunk validation policy = %#v, want %#v", got, want)
}
resolvedLane := resolved.Steps[0].ArtifactLanes[0]
if got, want := resolvedLane.ExtractValidationPolicy, (ValidationPolicy{ProducerStructuralFailure: ProducerStructuralFailureRejectOutput, SemanticRejection: SemanticRejectionFailRun, ValidatorFailure: ValidatorFailureFailRun}); got != want {
t.Fatalf("extract validation policy = %#v, want %#v", got, want)
}
if got, want := resolvedLane.MergeValidationPolicy, (ValidationPolicy{ProducerStructuralFailure: ProducerStructuralFailureRejectOutput, SemanticRejection: SemanticRejectionRejectOutput, ValidatorFailure: ValidatorFailureWarnContinue}); got != want {
t.Fatalf("merge validation policy = %#v, want %#v", got, want)
}
if got, want := resolvedLane.NormalizeValidationPolicy, (ValidationPolicy{ProducerStructuralFailure: ProducerStructuralFailureRejectOutput, SemanticRejection: SemanticRejectionRejectOutput, ValidatorFailure: ValidatorFailureFailRun}); got != want {
t.Fatalf("normalize validation policy = %#v, want %#v", got, want)
}
*profile.ValidationPolicy.SemanticRejection = SemanticRejectionFailRun
if got := resolved.ConfiguredValidationPolicy.SemanticRejection; got == nil || *got != SemanticRejectionRejectOutput {
t.Fatalf("resolved configured policy aliases profile: %#v", resolved.ConfiguredValidationPolicy)
}
}
func TestResolvePipelineRejectsStructuralPolicyOverrideOnDeterministicProducer(t *testing.T) {
profile := baselineProfile()
profile.Chunk.ValidationPolicy = validationPolicyOverride(ProducerStructuralFailureRejectOutput, "", "")
_, err := ResolvePipeline(profile, ResolveOptions{}, newProfileCatalog(t))
if err == nil || !strings.Contains(err.Error(), "producer_structural_failure") || !strings.Contains(err.Error(), "deterministic") {
t.Fatalf("ResolvePipeline() error = %v, want deterministic structural-policy rejection", err)
}
}
func TestValidationPolicyChangesResolvedDigestAndRoundTripsBindings(t *testing.T) {
left := baselineProfile()
right := baselineProfile()
right.ValidationPolicy = validationPolicyOverride("", SemanticRejectionRejectOutput, "")
leftResolved, err := ResolvePipeline(left, ResolveOptions{}, newProfileCatalog(t))
if err != nil {
t.Fatalf("ResolvePipeline(left) error = %v", err)
}
rightResolved, err := ResolvePipeline(right, ResolveOptions{}, newProfileCatalog(t))
if err != nil {
t.Fatalf("ResolvePipeline(right) error = %v", err)
}
if leftResolved.Digest == rightResolved.Digest {
t.Fatal("validation policy change did not change resolved digest")
}
binding := ModuleBinding{Module: "producer", ValidationPolicy: validationPolicyOverride(ProducerStructuralFailureRejectOutput, SemanticRejectionRejectOutput, ValidatorFailureFailRun)}
encoded, err := json.Marshal(binding)
if err != nil {
t.Fatalf("marshal binding: %v", err)
}
var decoded ModuleBinding
if err := json.Unmarshal(encoded, &decoded); err != nil {
t.Fatalf("unmarshal binding: %v", err)
}
if decoded.ValidationPolicy == nil || *decoded.ValidationPolicy.ProducerStructuralFailure != ProducerStructuralFailureRejectOutput || *decoded.ValidationPolicy.SemanticRejection != SemanticRejectionRejectOutput || *decoded.ValidationPolicy.ValidatorFailure != ValidatorFailureFailRun {
t.Fatalf("round-tripped validation policy = %#v", decoded.ValidationPolicy)
}
cloned := cloneModuleBinding(binding)
*binding.ValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
if got := *cloned.ValidationPolicy.ValidatorFailure; got != ValidatorFailureFailRun {
t.Fatalf("cloned validation policy aliases binding: %q", got)
}
}
func validationPolicyOverride(structural ProducerStructuralFailureAction, semantic SemanticRejectionAction, validator ValidatorFailureAction) *ValidationPolicyOverride {
policy := &ValidationPolicyOverride{}
if structural != "" {
policy.ProducerStructuralFailure = &structural
}
if semantic != "" {
policy.SemanticRejection = &semantic
}
if validator != "" {
policy.ValidatorFailure = &validator
}
return policy
}

View File

@@ -95,6 +95,7 @@ func cloneModuleBinding(binding ModuleBinding) ModuleBinding {
binding.Module = strings.TrimSpace(binding.Module)
binding.LLMProfile = strings.TrimSpace(binding.LLMProfile)
binding.StructuredOutputRepairAttempts = cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts)
binding.ValidationPolicy = cloneValidationPolicyOverride(binding.ValidationPolicy)
binding.Options = cloneOptions(binding.Options)
if len(binding.References) > 0 {
references := make(map[string]ReferenceSource, len(binding.References))