Add validation policy configuration
This commit is contained in:
@@ -406,6 +406,7 @@ func validateRegistrySet(resolved ResolvedPipeline, registries Registries) error
|
||||
|
||||
func cloneResolvedPipeline(in ResolvedPipeline) ResolvedPipeline {
|
||||
out := in
|
||||
out.ConfiguredValidationPolicy = cloneValidationPolicyOverride(in.ConfiguredValidationPolicy)
|
||||
out.Input = cloneModuleBinding(in.Input)
|
||||
out.Chunk = cloneModuleBinding(in.Chunk)
|
||||
out.Output = cloneModuleBinding(in.Output)
|
||||
|
||||
@@ -24,6 +24,7 @@ type ModuleBinding struct {
|
||||
Module string `json:"module"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||
ValidationPolicy *ValidationPolicyOverride `json:"validation_policy,omitempty"`
|
||||
Retries int `json:"retries,omitempty"`
|
||||
Options map[string]any `json:"options,omitempty"`
|
||||
References map[string]ReferenceSource `json:"references,omitempty"`
|
||||
@@ -84,6 +85,7 @@ func (binding ModuleBinding) MarshalJSON() ([]byte, error) {
|
||||
Module string `json:"module"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||
ValidationPolicy *ValidationPolicyOverride `json:"validation_policy,omitempty"`
|
||||
Retries int `json:"retries,omitempty"`
|
||||
Options map[string]any `json:"options,omitempty"`
|
||||
References map[string]ReferenceSource `json:"references,omitempty"`
|
||||
@@ -93,6 +95,7 @@ func (binding ModuleBinding) MarshalJSON() ([]byte, error) {
|
||||
Module: binding.Module,
|
||||
LLMProfile: binding.LLMProfile,
|
||||
StructuredOutputRepairAttempts: binding.StructuredOutputRepairAttempts,
|
||||
ValidationPolicy: cloneValidationPolicyOverride(binding.ValidationPolicy),
|
||||
Retries: binding.Retries,
|
||||
Options: binding.Options,
|
||||
References: binding.References,
|
||||
@@ -112,6 +115,7 @@ func (binding *ModuleBinding) UnmarshalJSON(data []byte) error {
|
||||
Module string `json:"module"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||
ValidationPolicy *ValidationPolicyOverride `json:"validation_policy,omitempty"`
|
||||
Retries int `json:"retries,omitempty"`
|
||||
Options map[string]any `json:"options,omitempty"`
|
||||
References map[string]ReferenceSource `json:"references,omitempty"`
|
||||
@@ -125,6 +129,7 @@ func (binding *ModuleBinding) UnmarshalJSON(data []byte) error {
|
||||
Module: decoded.Module,
|
||||
LLMProfile: decoded.LLMProfile,
|
||||
StructuredOutputRepairAttempts: decoded.StructuredOutputRepairAttempts,
|
||||
ValidationPolicy: cloneValidationPolicyOverride(decoded.ValidationPolicy),
|
||||
Retries: decoded.Retries,
|
||||
Options: decoded.Options,
|
||||
References: decoded.References,
|
||||
@@ -158,6 +163,7 @@ type PipelineProfile struct {
|
||||
ID string `json:"id"`
|
||||
LLMProfile string `json:"llm_profile,omitempty"`
|
||||
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
|
||||
ValidationPolicy *ValidationPolicyOverride `json:"validation_policy,omitempty"`
|
||||
Input ModuleBinding `json:"input"`
|
||||
Chunk ModuleBinding `json:"chunk,omitempty"`
|
||||
Artifacts map[string]ArtifactLaneProfile `json:"artifacts"`
|
||||
@@ -198,23 +204,26 @@ type ResolvedReferenceTarget struct {
|
||||
}
|
||||
|
||||
type ResolvedArtifactLane struct {
|
||||
StepID string
|
||||
ID string
|
||||
ArtifactKind contracts.ArtifactKind `json:"artifact_kind,omitempty"`
|
||||
ArtifactSchemaID string `json:"artifact_schema_id,omitempty"`
|
||||
ArtifactSchemaName string `json:"artifact_schema_name,omitempty"`
|
||||
ArtifactSchemaVersion string `json:"artifact_schema_version,omitempty"`
|
||||
ArtifactSchemaDigest string `json:"artifact_schema_digest,omitempty"`
|
||||
Extract ModuleBinding
|
||||
ExtractExecutionClass contracts.ExecutionClass `json:"extract_execution_class"`
|
||||
Merge ModuleBinding
|
||||
MergeExecutionClass contracts.ExecutionClass `json:"merge_execution_class"`
|
||||
Normalize ModuleBinding
|
||||
NormalizeExecutionClass contracts.ExecutionClass `json:"normalize_execution_class"`
|
||||
Validators []ModuleBinding
|
||||
ExtractReferences ResolvedReferenceTarget `json:"extract_references"`
|
||||
MergeReferences ResolvedReferenceTarget `json:"merge_references"`
|
||||
NormalizeReferences ResolvedReferenceTarget `json:"normalize_references"`
|
||||
StepID string
|
||||
ID string
|
||||
ArtifactKind contracts.ArtifactKind `json:"artifact_kind,omitempty"`
|
||||
ArtifactSchemaID string `json:"artifact_schema_id,omitempty"`
|
||||
ArtifactSchemaName string `json:"artifact_schema_name,omitempty"`
|
||||
ArtifactSchemaVersion string `json:"artifact_schema_version,omitempty"`
|
||||
ArtifactSchemaDigest string `json:"artifact_schema_digest,omitempty"`
|
||||
Extract ModuleBinding
|
||||
ExtractExecutionClass contracts.ExecutionClass `json:"extract_execution_class"`
|
||||
ExtractValidationPolicy ValidationPolicy `json:"extract_validation_policy"`
|
||||
Merge ModuleBinding
|
||||
MergeExecutionClass contracts.ExecutionClass `json:"merge_execution_class"`
|
||||
MergeValidationPolicy ValidationPolicy `json:"merge_validation_policy"`
|
||||
Normalize ModuleBinding
|
||||
NormalizeExecutionClass contracts.ExecutionClass `json:"normalize_execution_class"`
|
||||
NormalizeValidationPolicy ValidationPolicy `json:"normalize_validation_policy"`
|
||||
Validators []ModuleBinding
|
||||
ExtractReferences ResolvedReferenceTarget `json:"extract_references"`
|
||||
MergeReferences ResolvedReferenceTarget `json:"merge_references"`
|
||||
NormalizeReferences ResolvedReferenceTarget `json:"normalize_references"`
|
||||
}
|
||||
|
||||
type ResolvedPipelineStep struct {
|
||||
@@ -237,17 +246,19 @@ type ResolvedValidator struct {
|
||||
}
|
||||
|
||||
type ResolvedPipeline struct {
|
||||
ID string
|
||||
Digest string
|
||||
Input ModuleBinding
|
||||
InputExecutionClass contracts.ExecutionClass `json:"input_execution_class"`
|
||||
Chunk ModuleBinding
|
||||
ChunkExecutionClass contracts.ExecutionClass `json:"chunk_execution_class"`
|
||||
ChunkReferences ResolvedReferenceTarget `json:"chunk_references"`
|
||||
Steps []ResolvedPipelineStep
|
||||
ValidatorChains []ResolvedValidatorChain `json:"validator_chains"`
|
||||
Output ModuleBinding
|
||||
OutputExecutionClass contracts.ExecutionClass `json:"output_execution_class"`
|
||||
ID string
|
||||
Digest string
|
||||
ConfiguredValidationPolicy *ValidationPolicyOverride `json:"configured_validation_policy,omitempty"`
|
||||
Input ModuleBinding
|
||||
InputExecutionClass contracts.ExecutionClass `json:"input_execution_class"`
|
||||
Chunk ModuleBinding
|
||||
ChunkExecutionClass contracts.ExecutionClass `json:"chunk_execution_class"`
|
||||
ChunkValidationPolicy ValidationPolicy `json:"chunk_validation_policy"`
|
||||
ChunkReferences ResolvedReferenceTarget `json:"chunk_references"`
|
||||
Steps []ResolvedPipelineStep
|
||||
ValidatorChains []ResolvedValidatorChain `json:"validator_chains"`
|
||||
Output ModuleBinding
|
||||
OutputExecutionClass contracts.ExecutionClass `json:"output_execution_class"`
|
||||
}
|
||||
|
||||
// AllArtifactLanes returns lanes in deterministic step order for read-only
|
||||
@@ -414,13 +425,14 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
|
||||
return ResolvedPipeline{}, err
|
||||
}
|
||||
resolved := ResolvedPipeline{
|
||||
ID: pipelineID,
|
||||
Input: input,
|
||||
InputExecutionClass: inputModuleSpec.ExecutionClass,
|
||||
Chunk: chunk,
|
||||
ChunkExecutionClass: chunkSpec.ExecutionClass,
|
||||
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences),
|
||||
Output: output,
|
||||
ID: pipelineID,
|
||||
ConfiguredValidationPolicy: cloneValidationPolicyOverride(profile.ValidationPolicy),
|
||||
Input: input,
|
||||
InputExecutionClass: inputModuleSpec.ExecutionClass,
|
||||
Chunk: chunk,
|
||||
ChunkExecutionClass: chunkSpec.ExecutionClass,
|
||||
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences),
|
||||
Output: output,
|
||||
}
|
||||
chunkValidatorChain, err := resolveValidatorChain(pipelineID, "", StageChunk, chunk.Module, chunk.Validators, "", nil, catalog)
|
||||
if err != nil {
|
||||
@@ -493,6 +505,9 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
|
||||
if err := applyEffectiveStructuredOutputRepairAttempts(&resolved, profile.StructuredOutputRepairAttempts); err != nil {
|
||||
return ResolvedPipeline{}, err
|
||||
}
|
||||
if err := applyEffectiveValidationPolicies(&resolved, profile.ValidationPolicy); err != nil {
|
||||
return ResolvedPipeline{}, err
|
||||
}
|
||||
if err := validateResolvedOptions(resolved, catalog, configuredLaneIDs); err != nil {
|
||||
return ResolvedPipeline{}, err
|
||||
}
|
||||
@@ -1328,6 +1343,7 @@ func resolveBinding(binding ModuleBinding, defaultModule string, referenceSlotLa
|
||||
Module: module,
|
||||
LLMProfile: llmProfile,
|
||||
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts),
|
||||
ValidationPolicy: cloneValidationPolicyOverride(binding.ValidationPolicy),
|
||||
Retries: binding.Retries,
|
||||
Options: cloneOptions(binding.Options),
|
||||
References: references,
|
||||
@@ -1456,6 +1472,72 @@ func applyEffectiveStructuredOutputRepairAttempts(resolved *ResolvedPipeline, pi
|
||||
return nil
|
||||
}
|
||||
|
||||
func applyEffectiveValidationPolicies(resolved *ResolvedPipeline, pipelinePolicy *ValidationPolicyOverride) error {
|
||||
apply := func(stage ModuleStage, laneID string, binding ModuleBinding, executionClass contracts.ExecutionClass) (ValidationPolicy, error) {
|
||||
binding.ValidationPolicy = cloneValidationPolicyOverride(binding.ValidationPolicy)
|
||||
if binding.ValidationPolicy != nil {
|
||||
if err := binding.ValidationPolicy.Validate(); err != nil {
|
||||
return ValidationPolicy{}, fmt.Errorf("pipeline %q %s validation_policy: %w", resolved.ID, stage, err)
|
||||
}
|
||||
if executionClass == contracts.ExecutionClassDeterministic && binding.ValidationPolicy.ProducerStructuralFailure != nil {
|
||||
if laneID == "" {
|
||||
return ValidationPolicy{}, fmt.Errorf("pipeline %q %s %q assigns producer_structural_failure to deterministic module", resolved.ID, stage, binding.Module)
|
||||
}
|
||||
return ValidationPolicy{}, fmt.Errorf("pipeline %q lane %q %s %q assigns producer_structural_failure to deterministic module", resolved.ID, laneID, stage, binding.Module)
|
||||
}
|
||||
}
|
||||
return ResolveValidationPolicy(binding.ValidationPolicy, pipelinePolicy), nil
|
||||
}
|
||||
|
||||
if pipelinePolicy != nil {
|
||||
if err := pipelinePolicy.Validate(); err != nil {
|
||||
return fmt.Errorf("pipeline %q validation_policy: %w", resolved.ID, err)
|
||||
}
|
||||
}
|
||||
if resolved.Input.ValidationPolicy != nil {
|
||||
return fmt.Errorf("pipeline %q input validation_policy is not supported", resolved.ID)
|
||||
}
|
||||
if resolved.Output.ValidationPolicy != nil {
|
||||
return fmt.Errorf("pipeline %q output validation_policy is not supported", resolved.ID)
|
||||
}
|
||||
for _, chain := range resolved.ValidatorChains {
|
||||
for _, validator := range chain.Validators {
|
||||
if validator.Binding.ValidationPolicy != nil {
|
||||
if chain.LaneID == "" {
|
||||
return fmt.Errorf("pipeline %q %s validator %q validation_policy is not supported", resolved.ID, chain.Stage, validator.Binding.Module)
|
||||
}
|
||||
return fmt.Errorf("pipeline %q lane %q %s validator %q validation_policy is not supported", resolved.ID, chain.LaneID, chain.Stage, validator.Binding.Module)
|
||||
}
|
||||
}
|
||||
}
|
||||
policy, err := apply(StageChunk, "", resolved.Chunk, resolved.ChunkExecutionClass)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resolved.ChunkValidationPolicy = policy
|
||||
for stepIndex := range resolved.Steps {
|
||||
for laneIndex := range resolved.Steps[stepIndex].ArtifactLanes {
|
||||
lane := &resolved.Steps[stepIndex].ArtifactLanes[laneIndex]
|
||||
policy, err = apply(StageExtract, lane.ID, lane.Extract, lane.ExtractExecutionClass)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lane.ExtractValidationPolicy = policy
|
||||
policy, err = apply(StageMerge, lane.ID, lane.Merge, lane.MergeExecutionClass)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lane.MergeValidationPolicy = policy
|
||||
policy, err = apply(StageNormalize, lane.ID, lane.Normalize, lane.NormalizeExecutionClass)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
lane.NormalizeValidationPolicy = policy
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func resolveBindings(bindings []ModuleBinding, defaultModule string, referenceSlotLabel string) ([]ModuleBinding, error) {
|
||||
if len(bindings) == 0 {
|
||||
return nil, nil
|
||||
@@ -1549,27 +1631,31 @@ func selectedArtifactLanes(pipelineID string, artifacts map[string]ArtifactLaneP
|
||||
|
||||
func resolvedPipelineDigest(resolved ResolvedPipeline) (string, error) {
|
||||
withoutDigest := struct {
|
||||
ID string
|
||||
Input ModuleBinding
|
||||
InputExecutionClass contracts.ExecutionClass
|
||||
Chunk ModuleBinding
|
||||
ChunkExecutionClass contracts.ExecutionClass
|
||||
ChunkReferences ResolvedReferenceTarget
|
||||
Steps []ResolvedPipelineStep
|
||||
ValidatorChains []ResolvedValidatorChain
|
||||
Output ModuleBinding
|
||||
OutputExecutionClass contracts.ExecutionClass
|
||||
ID string
|
||||
ConfiguredValidationPolicy *ValidationPolicyOverride
|
||||
Input ModuleBinding
|
||||
InputExecutionClass contracts.ExecutionClass
|
||||
Chunk ModuleBinding
|
||||
ChunkExecutionClass contracts.ExecutionClass
|
||||
ChunkValidationPolicy ValidationPolicy
|
||||
ChunkReferences ResolvedReferenceTarget
|
||||
Steps []ResolvedPipelineStep
|
||||
ValidatorChains []ResolvedValidatorChain
|
||||
Output ModuleBinding
|
||||
OutputExecutionClass contracts.ExecutionClass
|
||||
}{
|
||||
ID: resolved.ID,
|
||||
Input: resolved.Input,
|
||||
InputExecutionClass: resolved.InputExecutionClass,
|
||||
Chunk: resolved.Chunk,
|
||||
ChunkExecutionClass: resolved.ChunkExecutionClass,
|
||||
ChunkReferences: resolved.ChunkReferences,
|
||||
Steps: resolved.Steps,
|
||||
ValidatorChains: resolved.ValidatorChains,
|
||||
Output: resolved.Output,
|
||||
OutputExecutionClass: resolved.OutputExecutionClass,
|
||||
ID: resolved.ID,
|
||||
ConfiguredValidationPolicy: cloneValidationPolicyOverride(resolved.ConfiguredValidationPolicy),
|
||||
ChunkValidationPolicy: resolved.ChunkValidationPolicy,
|
||||
Input: resolved.Input,
|
||||
InputExecutionClass: resolved.InputExecutionClass,
|
||||
Chunk: resolved.Chunk,
|
||||
ChunkExecutionClass: resolved.ChunkExecutionClass,
|
||||
ChunkReferences: resolved.ChunkReferences,
|
||||
Steps: resolved.Steps,
|
||||
ValidatorChains: resolved.ValidatorChains,
|
||||
Output: resolved.Output,
|
||||
OutputExecutionClass: resolved.OutputExecutionClass,
|
||||
}
|
||||
encoded, err := json.Marshal(withoutDigest)
|
||||
if err != nil {
|
||||
|
||||
115
internal/framework/pipeline/validation_policy.go
Normal file
115
internal/framework/pipeline/validation_policy.go
Normal file
@@ -0,0 +1,115 @@
|
||||
package pipeline
|
||||
|
||||
import "fmt"
|
||||
|
||||
type ProducerStructuralFailureAction string
|
||||
|
||||
const (
|
||||
ProducerStructuralFailureFailRun ProducerStructuralFailureAction = "fail_run"
|
||||
ProducerStructuralFailureRejectOutput ProducerStructuralFailureAction = "reject_output"
|
||||
)
|
||||
|
||||
type SemanticRejectionAction string
|
||||
|
||||
const (
|
||||
SemanticRejectionFailRun SemanticRejectionAction = "fail_run"
|
||||
SemanticRejectionRejectOutput SemanticRejectionAction = "reject_output"
|
||||
)
|
||||
|
||||
type ValidatorFailureAction string
|
||||
|
||||
const (
|
||||
ValidatorFailureWarnContinue ValidatorFailureAction = "warn_continue"
|
||||
ValidatorFailureFailRun ValidatorFailureAction = "fail_run"
|
||||
)
|
||||
|
||||
// ValidationPolicyOverride records only explicitly configured terminal-policy
|
||||
// values. Nil fields inherit independently from the pipeline or application.
|
||||
type ValidationPolicyOverride struct {
|
||||
ProducerStructuralFailure *ProducerStructuralFailureAction `json:"producer_structural_failure,omitempty"`
|
||||
SemanticRejection *SemanticRejectionAction `json:"semantic_rejection,omitempty"`
|
||||
ValidatorFailure *ValidatorFailureAction `json:"validator_failure,omitempty"`
|
||||
}
|
||||
|
||||
// ValidationPolicy is the concrete terminal policy retained by each resolved
|
||||
// producer. It contains no inherited pointers.
|
||||
type ValidationPolicy struct {
|
||||
ProducerStructuralFailure ProducerStructuralFailureAction `json:"producer_structural_failure"`
|
||||
SemanticRejection SemanticRejectionAction `json:"semantic_rejection"`
|
||||
ValidatorFailure ValidatorFailureAction `json:"validator_failure"`
|
||||
}
|
||||
|
||||
func DefaultValidationPolicy() ValidationPolicy {
|
||||
return ValidationPolicy{
|
||||
ProducerStructuralFailure: ProducerStructuralFailureFailRun,
|
||||
SemanticRejection: SemanticRejectionFailRun,
|
||||
ValidatorFailure: ValidatorFailureWarnContinue,
|
||||
}
|
||||
}
|
||||
|
||||
func (override ValidationPolicyOverride) Validate() error {
|
||||
if override.ProducerStructuralFailure != nil {
|
||||
switch *override.ProducerStructuralFailure {
|
||||
case ProducerStructuralFailureFailRun, ProducerStructuralFailureRejectOutput:
|
||||
default:
|
||||
return fmt.Errorf("producer_structural_failure must be fail_run or reject_output")
|
||||
}
|
||||
}
|
||||
if override.SemanticRejection != nil {
|
||||
switch *override.SemanticRejection {
|
||||
case SemanticRejectionFailRun, SemanticRejectionRejectOutput:
|
||||
default:
|
||||
return fmt.Errorf("semantic_rejection must be fail_run or reject_output")
|
||||
}
|
||||
}
|
||||
if override.ValidatorFailure != nil {
|
||||
switch *override.ValidatorFailure {
|
||||
case ValidatorFailureWarnContinue, ValidatorFailureFailRun:
|
||||
default:
|
||||
return fmt.Errorf("validator_failure must be warn_continue or fail_run")
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func ResolveValidationPolicy(binding, pipeline *ValidationPolicyOverride) ValidationPolicy {
|
||||
resolved := DefaultValidationPolicy()
|
||||
applyValidationPolicyOverride(&resolved, pipeline)
|
||||
applyValidationPolicyOverride(&resolved, binding)
|
||||
return resolved
|
||||
}
|
||||
|
||||
func cloneValidationPolicyOverride(override *ValidationPolicyOverride) *ValidationPolicyOverride {
|
||||
if override == nil {
|
||||
return nil
|
||||
}
|
||||
out := *override
|
||||
if override.ProducerStructuralFailure != nil {
|
||||
value := *override.ProducerStructuralFailure
|
||||
out.ProducerStructuralFailure = &value
|
||||
}
|
||||
if override.SemanticRejection != nil {
|
||||
value := *override.SemanticRejection
|
||||
out.SemanticRejection = &value
|
||||
}
|
||||
if override.ValidatorFailure != nil {
|
||||
value := *override.ValidatorFailure
|
||||
out.ValidatorFailure = &value
|
||||
}
|
||||
return &out
|
||||
}
|
||||
|
||||
func applyValidationPolicyOverride(policy *ValidationPolicy, override *ValidationPolicyOverride) {
|
||||
if override == nil {
|
||||
return
|
||||
}
|
||||
if override.ProducerStructuralFailure != nil {
|
||||
policy.ProducerStructuralFailure = *override.ProducerStructuralFailure
|
||||
}
|
||||
if override.SemanticRejection != nil {
|
||||
policy.SemanticRejection = *override.SemanticRejection
|
||||
}
|
||||
if override.ValidatorFailure != nil {
|
||||
policy.ValidatorFailure = *override.ValidatorFailure
|
||||
}
|
||||
}
|
||||
103
internal/framework/pipeline/validation_policy_test.go
Normal file
103
internal/framework/pipeline/validation_policy_test.go
Normal file
@@ -0,0 +1,103 @@
|
||||
package pipeline
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestResolvePipelineAppliesValidationPolicyFieldByField(t *testing.T) {
|
||||
profile := baselineProfile()
|
||||
profile.ValidationPolicy = validationPolicyOverride(
|
||||
ProducerStructuralFailureRejectOutput,
|
||||
SemanticRejectionRejectOutput,
|
||||
ValidatorFailureFailRun,
|
||||
)
|
||||
profile.Chunk.ValidationPolicy = validationPolicyOverride("", "", ValidatorFailureWarnContinue)
|
||||
lane := profile.Artifacts["events"]
|
||||
lane.Extract.ValidationPolicy = validationPolicyOverride("", SemanticRejectionFailRun, "")
|
||||
lane.Merge.ValidationPolicy = validationPolicyOverride("", "", ValidatorFailureWarnContinue)
|
||||
profile.Artifacts["events"] = lane
|
||||
|
||||
resolved, err := ResolvePipeline(profile, ResolveOptions{}, newProfileCatalog(t))
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline() error = %v", err)
|
||||
}
|
||||
if got, want := resolved.ChunkValidationPolicy, (ValidationPolicy{ProducerStructuralFailure: ProducerStructuralFailureRejectOutput, SemanticRejection: SemanticRejectionRejectOutput, ValidatorFailure: ValidatorFailureWarnContinue}); got != want {
|
||||
t.Fatalf("chunk validation policy = %#v, want %#v", got, want)
|
||||
}
|
||||
resolvedLane := resolved.Steps[0].ArtifactLanes[0]
|
||||
if got, want := resolvedLane.ExtractValidationPolicy, (ValidationPolicy{ProducerStructuralFailure: ProducerStructuralFailureRejectOutput, SemanticRejection: SemanticRejectionFailRun, ValidatorFailure: ValidatorFailureFailRun}); got != want {
|
||||
t.Fatalf("extract validation policy = %#v, want %#v", got, want)
|
||||
}
|
||||
if got, want := resolvedLane.MergeValidationPolicy, (ValidationPolicy{ProducerStructuralFailure: ProducerStructuralFailureRejectOutput, SemanticRejection: SemanticRejectionRejectOutput, ValidatorFailure: ValidatorFailureWarnContinue}); got != want {
|
||||
t.Fatalf("merge validation policy = %#v, want %#v", got, want)
|
||||
}
|
||||
if got, want := resolvedLane.NormalizeValidationPolicy, (ValidationPolicy{ProducerStructuralFailure: ProducerStructuralFailureRejectOutput, SemanticRejection: SemanticRejectionRejectOutput, ValidatorFailure: ValidatorFailureFailRun}); got != want {
|
||||
t.Fatalf("normalize validation policy = %#v, want %#v", got, want)
|
||||
}
|
||||
|
||||
*profile.ValidationPolicy.SemanticRejection = SemanticRejectionFailRun
|
||||
if got := resolved.ConfiguredValidationPolicy.SemanticRejection; got == nil || *got != SemanticRejectionRejectOutput {
|
||||
t.Fatalf("resolved configured policy aliases profile: %#v", resolved.ConfiguredValidationPolicy)
|
||||
}
|
||||
}
|
||||
|
||||
func TestResolvePipelineRejectsStructuralPolicyOverrideOnDeterministicProducer(t *testing.T) {
|
||||
profile := baselineProfile()
|
||||
profile.Chunk.ValidationPolicy = validationPolicyOverride(ProducerStructuralFailureRejectOutput, "", "")
|
||||
_, err := ResolvePipeline(profile, ResolveOptions{}, newProfileCatalog(t))
|
||||
if err == nil || !strings.Contains(err.Error(), "producer_structural_failure") || !strings.Contains(err.Error(), "deterministic") {
|
||||
t.Fatalf("ResolvePipeline() error = %v, want deterministic structural-policy rejection", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidationPolicyChangesResolvedDigestAndRoundTripsBindings(t *testing.T) {
|
||||
left := baselineProfile()
|
||||
right := baselineProfile()
|
||||
right.ValidationPolicy = validationPolicyOverride("", SemanticRejectionRejectOutput, "")
|
||||
|
||||
leftResolved, err := ResolvePipeline(left, ResolveOptions{}, newProfileCatalog(t))
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline(left) error = %v", err)
|
||||
}
|
||||
rightResolved, err := ResolvePipeline(right, ResolveOptions{}, newProfileCatalog(t))
|
||||
if err != nil {
|
||||
t.Fatalf("ResolvePipeline(right) error = %v", err)
|
||||
}
|
||||
if leftResolved.Digest == rightResolved.Digest {
|
||||
t.Fatal("validation policy change did not change resolved digest")
|
||||
}
|
||||
|
||||
binding := ModuleBinding{Module: "producer", ValidationPolicy: validationPolicyOverride(ProducerStructuralFailureRejectOutput, SemanticRejectionRejectOutput, ValidatorFailureFailRun)}
|
||||
encoded, err := json.Marshal(binding)
|
||||
if err != nil {
|
||||
t.Fatalf("marshal binding: %v", err)
|
||||
}
|
||||
var decoded ModuleBinding
|
||||
if err := json.Unmarshal(encoded, &decoded); err != nil {
|
||||
t.Fatalf("unmarshal binding: %v", err)
|
||||
}
|
||||
if decoded.ValidationPolicy == nil || *decoded.ValidationPolicy.ProducerStructuralFailure != ProducerStructuralFailureRejectOutput || *decoded.ValidationPolicy.SemanticRejection != SemanticRejectionRejectOutput || *decoded.ValidationPolicy.ValidatorFailure != ValidatorFailureFailRun {
|
||||
t.Fatalf("round-tripped validation policy = %#v", decoded.ValidationPolicy)
|
||||
}
|
||||
cloned := cloneModuleBinding(binding)
|
||||
*binding.ValidationPolicy.ValidatorFailure = ValidatorFailureWarnContinue
|
||||
if got := *cloned.ValidationPolicy.ValidatorFailure; got != ValidatorFailureFailRun {
|
||||
t.Fatalf("cloned validation policy aliases binding: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func validationPolicyOverride(structural ProducerStructuralFailureAction, semantic SemanticRejectionAction, validator ValidatorFailureAction) *ValidationPolicyOverride {
|
||||
policy := &ValidationPolicyOverride{}
|
||||
if structural != "" {
|
||||
policy.ProducerStructuralFailure = &structural
|
||||
}
|
||||
if semantic != "" {
|
||||
policy.SemanticRejection = &semantic
|
||||
}
|
||||
if validator != "" {
|
||||
policy.ValidatorFailure = &validator
|
||||
}
|
||||
return policy
|
||||
}
|
||||
@@ -95,6 +95,7 @@ func cloneModuleBinding(binding ModuleBinding) ModuleBinding {
|
||||
binding.Module = strings.TrimSpace(binding.Module)
|
||||
binding.LLMProfile = strings.TrimSpace(binding.LLMProfile)
|
||||
binding.StructuredOutputRepairAttempts = cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts)
|
||||
binding.ValidationPolicy = cloneValidationPolicyOverride(binding.ValidationPolicy)
|
||||
binding.Options = cloneOptions(binding.Options)
|
||||
if len(binding.References) > 0 {
|
||||
references := make(map[string]ReferenceSource, len(binding.References))
|
||||
|
||||
Reference in New Issue
Block a user