eric/notarius
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add validation policy configuration

Browse Source
This commit is contained in:
Eric Rakestraw
2026-08-26 23:34:06 +00:00
parent 1c3da3e869
commit 9d0faabf61
14 changed files with 628 additions and 57 deletions
Download Patch File Download Diff File Expand all files Collapse all files

196
internal/framework/pipeline/profile.go
View File

@@ -24,6 +24,7 @@ type ModuleBinding struct {
Module string `json:"module"`
LLMProfile string `json:"llm_profile,omitempty"`
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
ValidationPolicy *ValidationPolicyOverride `json:"validation_policy,omitempty"`
Retries int `json:"retries,omitempty"`
Options map[string]any `json:"options,omitempty"`
References map[string]ReferenceSource `json:"references,omitempty"`
@@ -84,6 +85,7 @@ func (binding ModuleBinding) MarshalJSON() ([]byte, error) {
Module string `json:"module"`
LLMProfile string `json:"llm_profile,omitempty"`
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
ValidationPolicy *ValidationPolicyOverride `json:"validation_policy,omitempty"`
Retries int `json:"retries,omitempty"`
Options map[string]any `json:"options,omitempty"`
References map[string]ReferenceSource `json:"references,omitempty"`
@@ -93,6 +95,7 @@ func (binding ModuleBinding) MarshalJSON() ([]byte, error) {
Module: binding.Module,
LLMProfile: binding.LLMProfile,
StructuredOutputRepairAttempts: binding.StructuredOutputRepairAttempts,
ValidationPolicy: cloneValidationPolicyOverride(binding.ValidationPolicy),
Retries: binding.Retries,
Options: binding.Options,
References: binding.References,
@@ -112,6 +115,7 @@ func (binding *ModuleBinding) UnmarshalJSON(data []byte) error {
Module string `json:"module"`
LLMProfile string `json:"llm_profile,omitempty"`
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
ValidationPolicy *ValidationPolicyOverride `json:"validation_policy,omitempty"`
Retries int `json:"retries,omitempty"`
Options map[string]any `json:"options,omitempty"`
References map[string]ReferenceSource `json:"references,omitempty"`
@@ -125,6 +129,7 @@ func (binding *ModuleBinding) UnmarshalJSON(data []byte) error {
Module: decoded.Module,
LLMProfile: decoded.LLMProfile,
StructuredOutputRepairAttempts: decoded.StructuredOutputRepairAttempts,
ValidationPolicy: cloneValidationPolicyOverride(decoded.ValidationPolicy),
Retries: decoded.Retries,
Options: decoded.Options,
References: decoded.References,
@@ -158,6 +163,7 @@ type PipelineProfile struct {
ID string `json:"id"`
LLMProfile string `json:"llm_profile,omitempty"`
StructuredOutputRepairAttempts *int `json:"structured_output_repair_attempts,omitempty"`
ValidationPolicy *ValidationPolicyOverride `json:"validation_policy,omitempty"`
Input ModuleBinding `json:"input"`
Chunk ModuleBinding `json:"chunk,omitempty"`
Artifacts map[string]ArtifactLaneProfile `json:"artifacts"`
@@ -198,23 +204,26 @@ type ResolvedReferenceTarget struct {
}
type ResolvedArtifactLane struct {
StepID string
ID string
ArtifactKind contracts.ArtifactKind `json:"artifact_kind,omitempty"`
ArtifactSchemaID string `json:"artifact_schema_id,omitempty"`
ArtifactSchemaName string `json:"artifact_schema_name,omitempty"`
ArtifactSchemaVersion string `json:"artifact_schema_version,omitempty"`
ArtifactSchemaDigest string `json:"artifact_schema_digest,omitempty"`
Extract ModuleBinding
ExtractExecutionClass contracts.ExecutionClass `json:"extract_execution_class"`
Merge ModuleBinding
MergeExecutionClass contracts.ExecutionClass `json:"merge_execution_class"`
Normalize ModuleBinding
NormalizeExecutionClass contracts.ExecutionClass `json:"normalize_execution_class"`
Validators []ModuleBinding
ExtractReferences ResolvedReferenceTarget `json:"extract_references"`
MergeReferences ResolvedReferenceTarget `json:"merge_references"`
NormalizeReferences ResolvedReferenceTarget `json:"normalize_references"`
StepID string
ID string
ArtifactKind contracts.ArtifactKind `json:"artifact_kind,omitempty"`
ArtifactSchemaID string `json:"artifact_schema_id,omitempty"`
ArtifactSchemaName string `json:"artifact_schema_name,omitempty"`
ArtifactSchemaVersion string `json:"artifact_schema_version,omitempty"`
ArtifactSchemaDigest string `json:"artifact_schema_digest,omitempty"`
Extract ModuleBinding
ExtractExecutionClass contracts.ExecutionClass `json:"extract_execution_class"`
ExtractValidationPolicy ValidationPolicy `json:"extract_validation_policy"`
Merge ModuleBinding
MergeExecutionClass contracts.ExecutionClass `json:"merge_execution_class"`
MergeValidationPolicy ValidationPolicy `json:"merge_validation_policy"`
Normalize ModuleBinding
NormalizeExecutionClass contracts.ExecutionClass `json:"normalize_execution_class"`
NormalizeValidationPolicy ValidationPolicy `json:"normalize_validation_policy"`
Validators []ModuleBinding
ExtractReferences ResolvedReferenceTarget `json:"extract_references"`
MergeReferences ResolvedReferenceTarget `json:"merge_references"`
NormalizeReferences ResolvedReferenceTarget `json:"normalize_references"`
}
type ResolvedPipelineStep struct {
@@ -237,17 +246,19 @@ type ResolvedValidator struct {
}
type ResolvedPipeline struct {
ID string
Digest string
Input ModuleBinding
InputExecutionClass contracts.ExecutionClass `json:"input_execution_class"`
Chunk ModuleBinding
ChunkExecutionClass contracts.ExecutionClass `json:"chunk_execution_class"`
ChunkReferences ResolvedReferenceTarget `json:"chunk_references"`
Steps []ResolvedPipelineStep
ValidatorChains []ResolvedValidatorChain `json:"validator_chains"`
Output ModuleBinding
OutputExecutionClass contracts.ExecutionClass `json:"output_execution_class"`
ID string
Digest string
ConfiguredValidationPolicy *ValidationPolicyOverride `json:"configured_validation_policy,omitempty"`
Input ModuleBinding
InputExecutionClass contracts.ExecutionClass `json:"input_execution_class"`
Chunk ModuleBinding
ChunkExecutionClass contracts.ExecutionClass `json:"chunk_execution_class"`
ChunkValidationPolicy ValidationPolicy `json:"chunk_validation_policy"`
ChunkReferences ResolvedReferenceTarget `json:"chunk_references"`
Steps []ResolvedPipelineStep
ValidatorChains []ResolvedValidatorChain `json:"validator_chains"`
Output ModuleBinding
OutputExecutionClass contracts.ExecutionClass `json:"output_execution_class"`
}
// AllArtifactLanes returns lanes in deterministic step order for read-only
@@ -414,13 +425,14 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
return ResolvedPipeline{}, err
}
resolved := ResolvedPipeline{
ID: pipelineID,
Input: input,
InputExecutionClass: inputModuleSpec.ExecutionClass,
Chunk: chunk,
ChunkExecutionClass: chunkSpec.ExecutionClass,
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences),
Output: output,
ID: pipelineID,
ConfiguredValidationPolicy: cloneValidationPolicyOverride(profile.ValidationPolicy),
Input: input,
InputExecutionClass: inputModuleSpec.ExecutionClass,
Chunk: chunk,
ChunkExecutionClass: chunkSpec.ExecutionClass,
ChunkReferences: referenceTarget(StageChunk, "", chunk.Module, chunkReferences),
Output: output,
}
chunkValidatorChain, err := resolveValidatorChain(pipelineID, "", StageChunk, chunk.Module, chunk.Validators, "", nil, catalog)
if err != nil {
@@ -493,6 +505,9 @@ func ResolvePipeline(profile PipelineProfile, options ResolveOptions, catalog Mo
if err := applyEffectiveStructuredOutputRepairAttempts(&resolved, profile.StructuredOutputRepairAttempts); err != nil {
return ResolvedPipeline{}, err
}
if err := applyEffectiveValidationPolicies(&resolved, profile.ValidationPolicy); err != nil {
return ResolvedPipeline{}, err
}
if err := validateResolvedOptions(resolved, catalog, configuredLaneIDs); err != nil {
return ResolvedPipeline{}, err
}
@@ -1328,6 +1343,7 @@ func resolveBinding(binding ModuleBinding, defaultModule string, referenceSlotLa
Module: module,
LLMProfile: llmProfile,
StructuredOutputRepairAttempts: cloneStructuredOutputRepairAttempts(binding.StructuredOutputRepairAttempts),
ValidationPolicy: cloneValidationPolicyOverride(binding.ValidationPolicy),
Retries: binding.Retries,
Options: cloneOptions(binding.Options),
References: references,
@@ -1456,6 +1472,72 @@ func applyEffectiveStructuredOutputRepairAttempts(resolved *ResolvedPipeline, pi
return nil
}
func applyEffectiveValidationPolicies(resolved *ResolvedPipeline, pipelinePolicy *ValidationPolicyOverride) error {
apply := func(stage ModuleStage, laneID string, binding ModuleBinding, executionClass contracts.ExecutionClass) (ValidationPolicy, error) {
binding.ValidationPolicy = cloneValidationPolicyOverride(binding.ValidationPolicy)
if binding.ValidationPolicy != nil {
if err := binding.ValidationPolicy.Validate(); err != nil {
return ValidationPolicy{}, fmt.Errorf("pipeline %q %s validation_policy: %w", resolved.ID, stage, err)
}
if executionClass == contracts.ExecutionClassDeterministic && binding.ValidationPolicy.ProducerStructuralFailure != nil {
if laneID == "" {
return ValidationPolicy{}, fmt.Errorf("pipeline %q %s %q assigns producer_structural_failure to deterministic module", resolved.ID, stage, binding.Module)
}
return ValidationPolicy{}, fmt.Errorf("pipeline %q lane %q %s %q assigns producer_structural_failure to deterministic module", resolved.ID, laneID, stage, binding.Module)
}
}
return ResolveValidationPolicy(binding.ValidationPolicy, pipelinePolicy), nil
}
if pipelinePolicy != nil {
if err := pipelinePolicy.Validate(); err != nil {
return fmt.Errorf("pipeline %q validation_policy: %w", resolved.ID, err)
}
}
if resolved.Input.ValidationPolicy != nil {
return fmt.Errorf("pipeline %q input validation_policy is not supported", resolved.ID)
}
if resolved.Output.ValidationPolicy != nil {
return fmt.Errorf("pipeline %q output validation_policy is not supported", resolved.ID)
}
for _, chain := range resolved.ValidatorChains {
for _, validator := range chain.Validators {
if validator.Binding.ValidationPolicy != nil {
if chain.LaneID == "" {
return fmt.Errorf("pipeline %q %s validator %q validation_policy is not supported", resolved.ID, chain.Stage, validator.Binding.Module)
}
return fmt.Errorf("pipeline %q lane %q %s validator %q validation_policy is not supported", resolved.ID, chain.LaneID, chain.Stage, validator.Binding.Module)
}
}
}
policy, err := apply(StageChunk, "", resolved.Chunk, resolved.ChunkExecutionClass)
if err != nil {
return err
}
resolved.ChunkValidationPolicy = policy
for stepIndex := range resolved.Steps {
for laneIndex := range resolved.Steps[stepIndex].ArtifactLanes {
lane := &resolved.Steps[stepIndex].ArtifactLanes[laneIndex]
policy, err = apply(StageExtract, lane.ID, lane.Extract, lane.ExtractExecutionClass)
if err != nil {
return err
}
lane.ExtractValidationPolicy = policy
policy, err = apply(StageMerge, lane.ID, lane.Merge, lane.MergeExecutionClass)
if err != nil {
return err
}
lane.MergeValidationPolicy = policy
policy, err = apply(StageNormalize, lane.ID, lane.Normalize, lane.NormalizeExecutionClass)
if err != nil {
return err
}
lane.NormalizeValidationPolicy = policy
}
}
return nil
}
func resolveBindings(bindings []ModuleBinding, defaultModule string, referenceSlotLabel string) ([]ModuleBinding, error) {
if len(bindings) == 0 {
return nil, nil
@@ -1549,27 +1631,31 @@ func selectedArtifactLanes(pipelineID string, artifacts map[string]ArtifactLaneP
func resolvedPipelineDigest(resolved ResolvedPipeline) (string, error) {
withoutDigest := struct {
ID string
Input ModuleBinding
InputExecutionClass contracts.ExecutionClass
Chunk ModuleBinding
ChunkExecutionClass contracts.ExecutionClass
ChunkReferences ResolvedReferenceTarget
Steps []ResolvedPipelineStep
ValidatorChains []ResolvedValidatorChain
Output ModuleBinding
OutputExecutionClass contracts.ExecutionClass
ID string
ConfiguredValidationPolicy *ValidationPolicyOverride
Input ModuleBinding
InputExecutionClass contracts.ExecutionClass
Chunk ModuleBinding
ChunkExecutionClass contracts.ExecutionClass
ChunkValidationPolicy ValidationPolicy
ChunkReferences ResolvedReferenceTarget
Steps []ResolvedPipelineStep
ValidatorChains []ResolvedValidatorChain
Output ModuleBinding
OutputExecutionClass contracts.ExecutionClass
}{
ID: resolved.ID,
Input: resolved.Input,
InputExecutionClass: resolved.InputExecutionClass,
Chunk: resolved.Chunk,
ChunkExecutionClass: resolved.ChunkExecutionClass,
ChunkReferences: resolved.ChunkReferences,
Steps: resolved.Steps,
ValidatorChains: resolved.ValidatorChains,
Output: resolved.Output,
OutputExecutionClass: resolved.OutputExecutionClass,
ID: resolved.ID,
ConfiguredValidationPolicy: cloneValidationPolicyOverride(resolved.ConfiguredValidationPolicy),
ChunkValidationPolicy: resolved.ChunkValidationPolicy,
Input: resolved.Input,
InputExecutionClass: resolved.InputExecutionClass,
Chunk: resolved.Chunk,
ChunkExecutionClass: resolved.ChunkExecutionClass,
ChunkReferences: resolved.ChunkReferences,
Steps: resolved.Steps,
ValidatorChains: resolved.ValidatorChains,
Output: resolved.Output,
OutputExecutionClass: resolved.OutputExecutionClass,
}
encoded, err := json.Marshal(withoutDigest)
if err != nil {